a working version matched with GSI

CMakeLists.txt

@@ -80,8 +80,8 @@ if(BUILD_GDASBUNDLE)
   option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
   ecbuild_bundle( PROJECT ioda  GIT "https://github.com/jcsda/ioda.git" BRANCH develop  )
   option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
-  ecbuild_bundle( PROJECT ufo   GIT "https://github.com/jcsda/ufo.git" BRANCH develop )
-
+  # ecbuild_bundle( PROJECT ufo   GIT "https://github.com/jcsda/ufo.git" BRANCH develop )
+  ecbuild_bundle( PROJECT ufo   GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/crtm )
 # FMS and FV3 dynamical core
   ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable )
   ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable )

parm/atm/obs/config/amsua_n19.yaml

@@ -18,6 +18,7 @@ obs operator:
   Absorbers: [H2O,O3]
   Clouds: [Water, Ice]
   Cloud_Fraction: 1.0
+  Cloud_Seeding: true
   obs options:
     Sensor_ID: &Sensor_ID amsua_n19
     EndianType: little_endian
@@ -100,6 +101,48 @@ obs post filters:
         clwret_ch314: 2
         clwret_types: [ObsValue]
 
+- filter: Variable Assignment
+  assignments:
+  - name: DerivedMetaData/surfaceParam
+    type: int
+    function:
+      name: IntObsFunction/Conditional
+      options:
+        defaultvalue: 0
+        firstmatchingcase: true
+        cases:
+        - where:
+          - variable:
+              name: GeoVaLs/water_area_fraction
+            minvalue: 0.99
+          value: 100
+        - where:
+          - variable:
+              name: GeoVaLs/land_area_fraction
+            minvalue: 0.99
+          value: 85
+        - where:
+          - variable:
+              name: GeoVaLs/ice_area_fraction
+            minvalue: 0.99
+          value: 90
+        - where:
+          - variable:
+              name: GeoVaLs/surface_snow_area_fraction
+            minvalue: 0.99
+          value: 85
+
+- filter: Variable Assignment
+  assignments:
+  - name: DerivedMetaData/thinningCriteria
+    type: int
+    function:
+      name: IntObsFunction/Arithmetic
+      options:
+        variables:
+        - name: DerivedMetaData/surfaceParam
+        coefs: [1]
+
 # Calculate CLW retrieved from background 
 - filter: Variable Assignment 
   assignments:
@@ -246,6 +289,7 @@ obs post filters:
   horizontal_mesh: 145
   use_reduced_horizontal_grid: true
   distance_norm: geodesic
+  priority_variable: DerivedMetaData/thinningCriteria
 #  round_horizontal_bin_count_to_nearest: true
 #  partition_longitude_bins_using_mesh: true
   actions:

parm/atm/variational/3dvar_dripcg.yaml

@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-Var
-  window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-  window length: $(ATM_WINDOW_LENGTH)
+  time window:
+    begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+    length: $(ATM_WINDOW_LENGTH)
   analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
   geometry:
     fms initialization:
@@ -33,7 +34,7 @@ variational:
   minimizer:
     algorithm: DRIPCG
   iterations:
-  - ninner: 2
+  - ninner: 1
     gradient norm reduction: 1e-10
     test: on
     geometry:
@@ -50,40 +51,6 @@ variational:
       field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml
     diagnostics:
       departures: bkgmob
-  - ninner: 4
-    gradient norm reduction: 1e-10
-    test: on
-    geometry:
-      fms initialization:
-        namelist filename: ./fv3jedi/fmsmpp.nml
-        field table filename: ./fv3jedi/field_table
-      akbk: ./fv3jedi/akbk.nc4
-      layout:
-      - $(layout_x)
-      - $(layout_y)
-      npx: $(npx_anl)
-      npy: $(npy_anl)
-      npz: $(npz_anl)
-      field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml
-    diagnostics:
-      departures: bkgmob1
 final:
   diagnostics:
     departures: anlmob
-  increment:
-    output:
-      filetype: auxgrid
-      gridtype: gaussian
-      filename: ./anl/atminc.
-    geometry:
-      fms initialization:
-        namelist filename: ./fv3jedi/fmsmpp.nml
-        field table filename: ./fv3jedi/field_table
-      akbk: ./fv3jedi/akbk.nc4
-      layout:
-      - $(layout_x)
-      - $(layout_y)
-      npx: $(npx_anl)
-      npy: $(npy_anl)
-      npz: $(npz_anl)
-      field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml

ush/ioda/bufr2ioda/antcorr_application.py

@@ -2,15 +2,17 @@
 import os
 import numpy as np
 
-INVALID = -1.0
+R1000 = 1000.0
+R1000000 = 1000000.0
+INVALID = R1000
 # Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE
 # Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520
 TSPACE = 2.7253
 
 
 class ACCoeff:
-    def __init__(self, ac_dir, sat_id='metop-c'):
-        file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '_v2.ACCoeff.nc')
+    def __init__(self, ac_dir, sat_id='n19'):
+        file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '.ACCoeff.nc')
         nc_file = nc.Dataset(file_name)
         self.n_fovs = len(nc_file.dimensions['n_FOVs'])
         self.n_channels = len(nc_file.dimensions['n_Channels'])
@@ -25,7 +27,10 @@ def remove_ant_corr(i, ac, ifov, t):
     # AC:             Structure containing the antenna correction coefficients for the sensor of interest.
     # iFOV:           The FOV index for a scanline of the sensor of interest.
     # T:              On input, this argument contains the brightness
+
+    # print(f't before corr: {t[:100]}')
     t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov]
+    # print(f't after corr: {t[:100]}')
     t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID]
     return t
 

ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py

@@ -5,23 +5,22 @@
 import os
 from combine_base import Bufr2IodaBase, CPP
 from wxflow import Logger
-from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr
+from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000
 from utils import timing_decorator, nc_merge
 
 logger = Logger(os.path.basename(__file__), level='INFO')
 
-R1000 = 1000.0
-R1000000 = 1000000.0
-
 BACKEND = CPP
 
-AMSUA_TYPE_CHANGE_DATETIME = "2000120000"
+AMSUA_TYPE_CHANGE_DATETIME = "2023120000"
 
-YAML_NORMAL = True  #  current as normal
+BAMUA = '1BAMUA'
+ESAMUA  = 'ESAMUA'
 
+YAML_NORMAL = True  # current as normal
 
-class Bufr2IodaAmusa(Bufr2IodaBase):
 
+class Bufr2IodaAmusa(Bufr2IodaBase):
     def __init__(self, yaml_order, *args, **kwargs):
         self.yaml_order = yaml_order
         super().__init__(*args, **kwargs)
@@ -34,7 +33,6 @@ def get_yaml_file(self):
 
 
 class Bufr2IodaAmusaChange(Bufr2IodaAmusa):
-
     def get_yaml_file(self):
         if self.yaml_order:
             return self.config['yaml_file'][1]
@@ -59,7 +57,9 @@ def re_map_variable(self, container):
                     ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id)
                 else:
                     ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id)
+                logger.info(f'ta before correction1: {ta[:100, :]}')
                 tb = self.apply_corr(sat_id, ta, ifov)
+                logger.info(f'tb after correction1: {tb[:100, :]}')
                 self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id)
 
     def apply_corr(self, sat_id, ta, ifov):
@@ -70,12 +70,16 @@ def apply_corr(self, sat_id, ta, ifov):
                 # Convert antenna temperature to brightness temperature
                 ifov = ifov.astype(int) - 1
                 for i in range(ta.shape[1]):
+                    logger.info(f'inside loop for allpy ta to tb: i = {i}')
                     x = ta[:, i]
+                    # logger.info(f'ta before correction: {x[:100]}')
                     if self.yaml_order:
-                        apply_ant_corr(i, ac, ifov, x)
+                        x = apply_ant_corr(i, ac, ifov, x)
                     else:
-                        remove_ant_corr(i, ac, ifov, x)
-                    x[x > R1000] = R1000000
+                        x = remove_ant_corr(i, ac, ifov, x)
+                    # logger.info(f'ta after correction: {x[:100]}')
+                    x[x >= R1000] = R1000000
+                    ta[:, i] = x
         else:
             pass  # TODO after know how to set llll
         return ta
@@ -118,10 +122,12 @@ def merge(amsua_files, splits):
         yaml_order = YAML_NORMAL
     else:
         yaml_order = not YAML_NORMAL
-
-    for sat_type in ['a', 'e']:
+    logger.info(f'yaml order is {yaml_order}')
+    BAMUA = '1BAMUA'
+    ESAMUA = 'ESAMUA'
+    for sat_type in [BAMUA, ESAMUA]:
         logger.info(f'Processing sat type: {sat_type}')
-        if sat_type == 'a':
+        if sat_type == BAMUA:
             convert = Bufr2IodaAmusa(yaml_order, args.config, backend=BACKEND)
         else:
             convert = Bufr2IodaAmusaChange(yaml_order, args.config, backend=BACKEND)