diff --git a/parm/aero/hofx/hofx_nomodel.yaml b/parm/aero/hofx/hofx_nomodel.yaml index 37a0b1c57..81516f0e4 100644 --- a/parm/aero/hofx/hofx_nomodel.yaml +++ b/parm/aero/hofx/hofx_nomodel.yaml @@ -11,7 +11,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml state: datapath: $(DATA)/bkg filetype: fms restart diff --git a/parm/aero/variational/3dvar_gfs_aero.yaml b/parm/aero/variational/3dvar_gfs_aero.yaml index 752154323..af1e62de2 100644 --- a/parm/aero/variational/3dvar_gfs_aero.yaml +++ b/parm/aero/variational/3dvar_gfs_aero.yaml @@ -24,7 +24,7 @@ cost function: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml background: datapath: $(DATA)/bkg filetype: fms restart @@ -55,7 +55,7 @@ variational: npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - ninner: 35 @@ -72,7 +72,7 @@ variational: npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob1 final: @@ -90,7 +90,7 @@ final: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml output: datapath: $(DATA)/anl prefix: aeroinc diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml index 14b6c0e66..3c6e79004 100644 --- a/parm/atm/hofx/hofx4d.yaml +++ b/parm/atm/hofx/hofx4d.yaml @@ -12,7 +12,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml initial condition: datapath: $(DATA)/bkg filetype: fms restart diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml index f980d53be..3f442d517 100644 --- a/parm/atm/hofx/hofx_nomodel.yaml +++ b/parm/atm/hofx/hofx_nomodel.yaml @@ -11,7 +11,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-restart.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml state: datapath: $(DATA)/bkg filetype: fms restart diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml index d4ee73bf1..a84eefcee 100644 --- a/parm/atm/lgetkf/lgetkf.yaml +++ b/parm/atm/lgetkf/lgetkf.yaml @@ -9,7 +9,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: ./fv3jedi/gfs-restart.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}' window length: $(ATM_WINDOW_LENGTH) increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr] diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml index 4563f45ac..abd945842 100644 --- a/parm/atm/variational/3dvar_dripcg.yaml +++ b/parm/atm/variational/3dvar_dripcg.yaml @@ -14,7 +14,7 @@ cost function: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: ./fv3jedi/gfs-restart.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml background: datapath: ./bkg filetype: fms restart @@ -47,7 +47,7 @@ variational: npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) - field metadata override: ./fv3jedi/gfs-restart.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob - ninner: 4 @@ -64,7 +64,7 @@ variational: npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) - field metadata override: ./fv3jedi/gfs-restart.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diagnostics: departures: bkgmob1 final: @@ -86,4 +86,4 @@ final: npx: $(npx_anl) npy: $(npy_anl) npz: $(npz_anl) - field metadata override: ./fv3jedi/gfs-restart.yaml + field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml diff --git a/parm/io/fv3jedi_fieldmetadata_history.yaml b/parm/io/fv3jedi_fieldmetadata_history.yaml new file mode 100644 index 000000000..6c9009527 --- /dev/null +++ b/parm/io/fv3jedi_fieldmetadata_history.yaml @@ -0,0 +1,68 @@ +field metadata: + +- long name: eastward_wind + io name: ugrd + +- long name: northward_wind + io name: vgrd + +- long name: specific_humidity + io name: spfh + +- long name: air_temperature + io name: tmp + +- long name: cloud_liquid_water + io name: clwmr + +- long name: cloud_liquid_ice + io name: icmr + +- long name: snow_water + io name: snmr + +- long name: rain_water + io name: rwmr + +- long name: graupel + io name: grle + +- long name: cloud_ice_number_concentration + io name: nccice + +- long name: rain_number_concentration + io name: nconrd + +- long name: ozone_mass_mixing_ratio + io name: o3mr + +- long name: air_pressure_thickness + io name: dpres + +- long name: layer_thickness + io name: delz + +- long name: upward_air_velocity + io name: dzdt + +- long name: surface_pressure + io name: pressfc + +- long name: sfc_geopotential_height_times_grav + io name: hgtsfc + +- long name: u_component_of_native_D_grid_wind + io name: ud + +- long name: v_component_of_native_D_grid_wind + io name: vd + +- long name: stc + io name: soilt + +- long name: soilMoistureVolumetric + io name: soilw + +# Dummy IO name since stc and soilt are different in JEDI but same in UFS +- long name: soilt + io name: dummy_io_name diff --git a/parm/io/fv3jedi_fieldmetadata_restart.yaml b/parm/io/fv3jedi_fieldmetadata_restart.yaml new file mode 100644 index 000000000..d4a4a3a09 --- /dev/null +++ b/parm/io/fv3jedi_fieldmetadata_restart.yaml @@ -0,0 +1,109 @@ +field metadata: + +- long name: u_component_of_native_D_grid_wind + io name: u + +- long name: v_component_of_native_D_grid_wind + io name: v + +- long name: air_temperature + io name: T + +- long name: air_pressure_thickness + io name: DELP + +- long name: layer_thickness + io name: DZ + +- long name: stc + io file: surface + +- long name: soilMoistureVolumetric + io name: smc + io file: surface + +- long name: slmsk + interpolation type: nearest + +- long name: totalSnowDepth + io name: snodl + interpolation type: nearest + +- long name: vtype + interpolation type: nearest + +- long name: fraction_of_ice + io name: fice + interpolation type: nearest + +- long name: mass_fraction_of_dust001_in_air + io name: dust1 + units: ugkg-1 + +- long name: mass_fraction_of_dust002_in_air + io name: dust2 + units: ugkg-1 + +- long name: mass_fraction_of_dust003_in_air + io name: dust3 + units: ugkg-1 + +- long name: mass_fraction_of_dust004_in_air + io name: dust4 + units: ugkg-1 + +- long name: mass_fraction_of_dust005_in_air + io name: dust5 + units: ugkg-1 + +- long name: mass_fraction_of_sea_salt001_in_air + io name: seas1 + units: ugkg-1 + +- long name: mass_fraction_of_sea_salt002_in_air + io name: seas2 + units: ugkg-1 + +- long name: mass_fraction_of_sea_salt003_in_air + io name: seas3 + units: ugkg-1 + +- long name: mass_fraction_of_sea_salt004_in_air + io name: seas4 + units: ugkg-1 + +- long name: mass_fraction_of_sea_salt005_in_air + io name: seas5 + units: ugkg-1 + +- long name: mass_fraction_of_hydrophobic_black_carbon_in_air + io name: bc1 + units: ugkg-1 + +- long name: mass_fraction_of_hydrophilic_black_carbon_in_air + io name: bc2 + units: ugkg-1 + +- long name: mass_fraction_of_hydrophobic_organic_carbon_in_air + io name: oc1 + units: ugkg-1 + +- long name: mass_fraction_of_hydrophilic_organic_carbon_in_air + io name: oc2 + units: ugkg-1 + +- long name: mass_fraction_of_sulfate_in_air + io name: so4 + units: ugkg-1 + +- long name: mass_fraction_of_nitrate001_in_air + io name: no3an1 + units: ugkg-1 + +- long name: mass_fraction_of_nitrate002_in_air + io name: no3an2 + units: ugkg-1 + +- long name: mass_fraction_of_nitrate003_in_air + io name: no3an3 + units: ugkg-1 diff --git a/parm/land/hofx/hofx_nomodel.yaml b/parm/land/hofx/hofx_nomodel.yaml index a767a0e28..c0a9bc8d2 100644 --- a/parm/land/hofx/hofx_nomodel.yaml +++ b/parm/land/hofx/hofx_nomodel.yaml @@ -11,7 +11,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-land.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml time invariant fields: state fields: datetime: '{{ LAND_WINDOW_BEGIN | to_isotime }}' diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index 11db7a94d..b1310504c 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -9,7 +9,7 @@ geometry: npx: $(npx_ges) npy: $(npy_ges) npz: $(npz_ges) - field metadata override: $(DATA)/fv3jedi/gfs-land.yaml + field metadata override: $(DATA)/fv3jedi/fv3jedi_fieldmetadata_restart.yaml time invariant fields: state fields: datetime: '{{ LAND_WINDOW_BEGIN | to_isotime }}' diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 220f79c70..51e14f207 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -49,6 +49,7 @@ list(APPEND test_input ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_ref.yaml ${PROJECT_SOURCE_DIR}/test/testinput/check_yaml_keys_test.yaml ${PROJECT_SOURCE_DIR}/test/testinput/bufr_adpsfc.yaml + ${PROJECT_SOURCE_DIR}/test/testinput/bufr_adpsfc_snow.yaml ${PROJECT_SOURCE_DIR}/test/testinput/amsua_n19_ewok.yaml ${PROJECT_SOURCE_DIR}/test/testinput/temp_bufr_dbuoyprof.yaml ${PROJECT_SOURCE_DIR}/test/testinput/salt_bufr_dbuoyprof.yaml diff --git a/test/land/CMakeLists.txt b/test/land/CMakeLists.txt index 08745c707..813b80fcd 100644 --- a/test/land/CMakeLists.txt +++ b/test/land/CMakeLists.txt @@ -30,3 +30,8 @@ set_tests_properties(test_gdasapp_land_letkfoi_snowda PROPERTIES ENVIRONMENT "GDASAPP_TESTDATA=$ENV{GDASAPP_TESTDATA}") +# Test convert BUFR to IODA +add_test(NAME test_gdasapp_convert_bufr_adpsfc_snow + COMMAND ${PROJECT_SOURCE_DIR}/test/soca/test_bufr2ioda.sh ${PROJECT_BINARY_DIR} bufr_adpsfc_snow + WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/test/) + diff --git a/test/land/letkfoi_snowda.sh b/test/land/letkfoi_snowda.sh index aecf2f62c..e6ca740f0 100755 --- a/test/land/letkfoi_snowda.sh +++ b/test/land/letkfoi_snowda.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -xe ################################################ YY=2021 MM=03 @@ -64,7 +64,8 @@ fi # Prepare and Run JEDI ############################################ mkdir -p Data diags -ln -s ${project_binary_dir}/fv3-jedi/test/Data/fieldmetadata Data/fieldmetadata +mkdir -p Data/fieldmetadata +ln -s ${project_source_dir}/parm/io/fv3jedi_fieldmetadata_restart.yaml Data/fieldmetadata/. ln -s ${project_binary_dir}/fv3-jedi/test/Data/fv3files Data/fv3files ln -s ${project_source_dir}/ush/land/genYAML_output_letkfoi.yaml letkf_land.yaml ln -s ${OBSDIR}/snow_depth/GTS/202103/adpsfc_snow_2021032318.nc4 adpsfc_snow.nc4 diff --git a/test/testinput/bufr_adpsfc_snow.yaml b/test/testinput/bufr_adpsfc_snow.yaml new file mode 100644 index 000000000..c7f322a35 --- /dev/null +++ b/test/testinput/bufr_adpsfc_snow.yaml @@ -0,0 +1,85 @@ +# (C) Copyright 2021-2022 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "./testdata/gdas.t06z.adpsfc.tm00.bufr_d" + + exports: + variables: + # MetaData + timestamp: + datetime: + year: "*/YEAR[1]" + month: "*/MNTH[1]" + day: "*/DAYS[1]" + hour: "*/HOUR[1]" + minute: "*/MINU[1]" + latitude: + query: "[*/CLAT, */CLATH]" + longitude: + query: "[*/CLON, */CLONH]" + stationIdentification: + query: "*/RPID" + + height: + query: "[*/SELV, */HSMSL]" + + # ObsValue + totalSnowDepth: + query: "[*/SNWSQ1/TOSD, */MTRMSC/TOSD, */STGDSNDM/TOSD]" + transforms: + - scale: 1000.0 + filters: + - bounding: + variable: totalSnowDepth + upperBound: 10000000 + + ioda: + backend: netcdf + obsdataout: "./testoutput/gdas.t06z.adpsfc_snow.nc" + + variables: + + # MetaData + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + longName: "Datetime" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degree_north" + range: [-90, 90] + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degree_east" + range: [-180, 180] + + - name: "MetaData/height" + coordinates: "longitude latitude" + source: variables/height + longName: "Height of Station" + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Identification of Observing Location" + units: "m" + + # ObsValue + - name: "ObsValue/totalSnowDepth" + coordinates: "longitude latitude" + source: variables/totalSnowDepth + longName: "Total Snow Depth" + units: "mm" diff --git a/ush/land/genYAML_output_letkfoi.yaml b/ush/land/genYAML_output_letkfoi.yaml index b55c3618a..c9069f0e8 100644 --- a/ush/land/genYAML_output_letkfoi.yaml +++ b/ush/land/genYAML_output_letkfoi.yaml @@ -20,7 +20,7 @@ driver: save posterior mean increment: 'true' geometry: akbk: Data/fv3files/akbk127.nc4 - field metadata override: Data/fieldmetadata/gfs-land.yaml + field metadata override: Data/fieldmetadata/fv3jedi_fieldmetadata_restart.yaml fms initialization: field table filename: Data/fv3files/field_table namelist filename: Data/fv3files/fmsmpp.nml diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 989589a53..bed7a9b39 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -63,9 +63,9 @@ def gdas_fix(input_fix_dir, working_dir, config): os.path.join(config['fv3jedi_fix_dir'], 'field_table')) # link fieldmetadata # Note that the required data will be dependent on input file type (restart vs history, etc.) - ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'fv3jedi', - 'fieldmetadata', 'gfs-restart.yaml'), - os.path.join(config['fv3jedi_fieldmetadata_dir'], 'gfs-restart.yaml')) + gdasapp_parm = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'parm')) + ufsda.disk_utils.symlink(os.path.join(gdasapp_parm, 'io', 'fv3jedi_fieldmetadata_restart.yaml'), + os.path.join(config['fv3jedi_fieldmetadata_dir'], 'fv3jedi_fieldmetadata_restart.yaml')) # link CRTM coeff dir ufsda.disk_utils.symlink(os.path.join(input_fix_dir, 'crtm', '2.3.0'), config['CRTM_COEFF_DIR']) diff --git a/ush/ufsda/yamltools.py b/ush/ufsda/yamltools.py index 4207afcd6..021aed93e 100644 --- a/ush/ufsda/yamltools.py +++ b/ush/ufsda/yamltools.py @@ -135,7 +135,7 @@ def fv3atm_geom_dict(case, levs, ntiles, layout, io_layout): 'npy': str(case+1), 'npz': str(levs-1), 'ntiles': str(ntiles), - 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/gfs-restart.yaml' + 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/fv3jedi_fieldmetadata_restart.yaml' } return outdict @@ -154,7 +154,7 @@ def fv3land_geom_dict(case, levs, ntiles, layout, io_layout): 'npy': str(case+1), 'npz': str(levs-1), 'ntiles': str(ntiles), - 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/gfs-land.yaml', + 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/fv3jedi_fieldmetadata_restart.yaml', 'time invariant fields': { 'state fields': { @@ -184,7 +184,7 @@ def fv3aero_geom_dict(case, levs, ntiles, layout, io_layout): 'npy': str(case+1), 'npz': str(levs-1), 'ntiles': str(ntiles), - 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/gfs-aero.yaml', + 'field metadata override': '$(fv3jedi_fieldmetadata_dir)/fv3jedi_fieldmetadata_restart.yaml', } return outdict