diff --git a/CMakeLists.txt b/CMakeLists.txt index 2091893fd..5968b4b89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,8 +80,8 @@ if(BUILD_GDASBUNDLE) option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) - + # ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop ) + ecbuild_bundle( PROJECT ufo GIT "https://github.com/JCSDA-internal/ufo.git" BRANCH feature/crtm ) # FMS and FV3 dynamical core ecbuild_bundle( PROJECT fms GIT "https://github.com/jcsda/FMS.git" BRANCH release-stable ) ecbuild_bundle( PROJECT fv3 GIT "https://github.com/jcsda/GFDL_atmos_cubed_sphere.git" BRANCH release-stable ) diff --git a/parm/atm/obs/config/amsua_n19.yaml b/parm/atm/obs/config/amsua_n19.yaml index aa91d3f6c..f6342d467 100644 --- a/parm/atm/obs/config/amsua_n19.yaml +++ b/parm/atm/obs/config/amsua_n19.yaml @@ -18,6 +18,7 @@ obs operator: Absorbers: [H2O,O3] Clouds: [Water, Ice] Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &Sensor_ID amsua_n19 EndianType: little_endian @@ -100,6 +101,48 @@ obs post filters: clwret_ch314: 2 clwret_types: [ObsValue] +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 100 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 85 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 90 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 85 + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + # Calculate CLW retrieved from background - filter: Variable Assignment assignments: @@ -246,6 +289,7 @@ obs post filters: horizontal_mesh: 145 use_reduced_horizontal_grid: true distance_norm: geodesic + priority_variable: DerivedMetaData/thinningCriteria # round_horizontal_bin_count_to_nearest: true # partition_longitude_bins_using_mesh: true actions: diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index abd945842..2fa198924 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -1,7 +1,8 @@ cost function: cost type: 3D-Var - window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' - window length: $(ATM_WINDOW_LENGTH) + time window: + begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' + length: $(ATM_WINDOW_LENGTH) analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr] geometry: fms initialization: @@ -33,7 +34,7 @@ variational: minimizer: algorithm: DRIPCG iterations: - - ninner: 2 + - ninner: 1 gradient norm reduction: 1e-10 test: on geometry: @@ -50,40 +51,6 @@ variational: field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - - ninner: 4 - gradient norm reduction: 1e-10 - test: on - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml - diagnostics: - departures: bkgmob1 final: diagnostics: departures: anlmob - increment: - output: - filetype: auxgrid - gridtype: gaussian - filename: ./anl/atminc. - geometry: - fms initialization: - namelist filename: ./fv3jedi/fmsmpp.nml - field table filename: ./fv3jedi/field_table - akbk: ./fv3jedi/akbk.nc4 - layout: - - $(layout_x) - - $(layout_y) - npx: $(npx_anl) - npy: $(npy_anl) - npz: $(npz_anl) - field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc new file mode 100644 index 000000000..28b062add Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_metop-a.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc new file mode 100644 index 000000000..fdc91fc80 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_metop-b.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc new file mode 100644 index 000000000..bf3a9c0f6 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n15.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc new file mode 100644 index 000000000..efeef51c7 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n16.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc new file mode 100644 index 000000000..a50cc26e0 Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n17.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc new file mode 100644 index 000000000..72310903c Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n18.ACCoeff.nc differ diff --git a/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc b/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc new file mode 100644 index 000000000..bc36197de Binary files /dev/null and b/parm/ioda/bufr2ioda/amsua_n19.ACCoeff.nc differ diff --git a/ush/ioda/bufr2ioda/antcorr_application.py b/ush/ioda/bufr2ioda/antcorr_application.py index fec9a1430..096223643 100644 --- a/ush/ioda/bufr2ioda/antcorr_application.py +++ b/ush/ioda/bufr2ioda/antcorr_application.py @@ -2,15 +2,17 @@ import os import numpy as np -INVALID = -1.0 +R1000 = 1000.0 +R1000000 = 1000000.0 +INVALID = R1000 # Cosmic background temperature. Taken from Mather,J.C. et. al., 1999, "Calibrator Design for the COBE # Far-Infrared Absolute Spectrophotometer (FIRAS)"Astrophysical Journal, vol 512, pp 511-520 TSPACE = 2.7253 class ACCoeff: - def __init__(self, ac_dir, sat_id='metop-c'): - file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '_v2.ACCoeff.nc') + def __init__(self, ac_dir, sat_id='n19'): + file_name = os.path.join(ac_dir, 'amsua_' + sat_id + '.ACCoeff.nc') nc_file = nc.Dataset(file_name) self.n_fovs = len(nc_file.dimensions['n_FOVs']) self.n_channels = len(nc_file.dimensions['n_Channels']) @@ -25,7 +27,10 @@ def remove_ant_corr(i, ac, ifov, t): # AC: Structure containing the antenna correction coefficients for the sensor of interest. # iFOV: The FOV index for a scanline of the sensor of interest. # T: On input, this argument contains the brightness + + # print(f't before corr: {t[:100]}') t = ac.a_ep[i, ifov] * t + ac.a_sp[i, ifov] + # print(f't after corr: {t[:100]}') t[(ifov < 1) | (ifov > ac.n_fovs)] = [INVALID] return t diff --git a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py index 29e9ed83d..9f5440401 100755 --- a/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_combine_ncep_amsua.py @@ -5,23 +5,22 @@ import os from combine_base import Bufr2IodaBase, CPP from wxflow import Logger -from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr +from antcorr_application import ACCoeff, apply_ant_corr, remove_ant_corr, R1000, R1000000 from utils import timing_decorator, nc_merge logger = Logger(os.path.basename(__file__), level='INFO') -R1000 = 1000.0 -R1000000 = 1000000.0 - BACKEND = CPP -AMSUA_TYPE_CHANGE_DATETIME = "2000120000" +AMSUA_TYPE_CHANGE_DATETIME = "2023120000" -YAML_NORMAL = True # current as normal +BAMUA = '1BAMUA' +ESAMUA = 'ESAMUA' +YAML_NORMAL = True # current as normal -class Bufr2IodaAmusa(Bufr2IodaBase): +class Bufr2IodaAmusa(Bufr2IodaBase): def __init__(self, yaml_order, *args, **kwargs): self.yaml_order = yaml_order super().__init__(*args, **kwargs) @@ -34,7 +33,6 @@ def get_yaml_file(self): class Bufr2IodaAmusaChange(Bufr2IodaAmusa): - def get_yaml_file(self): if self.yaml_order: return self.config['yaml_file'][1] @@ -59,7 +57,9 @@ def re_map_variable(self, container): ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) else: ifov = self.get_container_variable(container, 'MetaData', 'sensorScanPosition', sat_id) + logger.info(f'ta before correction1: {ta[:100, :]}') tb = self.apply_corr(sat_id, ta, ifov) + logger.info(f'tb after correction1: {tb[:100, :]}') self.replace_container_variable(container, 'ObsValue', 'brightnessTemperature', tb, sat_id) def apply_corr(self, sat_id, ta, ifov): @@ -70,12 +70,16 @@ def apply_corr(self, sat_id, ta, ifov): # Convert antenna temperature to brightness temperature ifov = ifov.astype(int) - 1 for i in range(ta.shape[1]): + logger.info(f'inside loop for allpy ta to tb: i = {i}') x = ta[:, i] + # logger.info(f'ta before correction: {x[:100]}') if self.yaml_order: - apply_ant_corr(i, ac, ifov, x) + x = apply_ant_corr(i, ac, ifov, x) else: - remove_ant_corr(i, ac, ifov, x) - x[x > R1000] = R1000000 + x = remove_ant_corr(i, ac, ifov, x) + # logger.info(f'ta after correction: {x[:100]}') + x[x >= R1000] = R1000000 + ta[:, i] = x else: pass # TODO after know how to set llll return ta @@ -118,10 +122,12 @@ def merge(amsua_files, splits): yaml_order = YAML_NORMAL else: yaml_order = not YAML_NORMAL - - for sat_type in ['a', 'e']: + logger.info(f'yaml order is {yaml_order}') + BAMUA = '1BAMUA' + ESAMUA = 'ESAMUA' + for sat_type in [BAMUA, ESAMUA]: logger.info(f'Processing sat type: {sat_type}') - if sat_type == 'a': + if sat_type == BAMUA: convert = Bufr2IodaAmusa(yaml_order, args.config, backend=BACKEND) else: convert = Bufr2IodaAmusaChange(yaml_order, args.config, backend=BACKEND)