Merge branch 'develop' into feature/iodaconv-mpi

NOAA-EMC · Sep 25, 2023 · 32c87f0 · 32c87f0
2 parents 480479b + 97362b9
commit 32c87f0
Show file tree

Hide file tree

Showing 6 changed files with 265 additions and 2 deletions.
diff --git a/parm/atm/obs/testing/gps_bend.yaml b/parm/atm/obs/testing/gps_bend.yaml
@@ -0,0 +1,86 @@
+obs space:
+  name: gnssrobndnbam
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV gps_bend_obs_${CDATE}.nc4 
+    obsgrouping:
+      group variables: [ 'sequenceNumber' ]
+      sort variable: 'impactHeightRO'
+      sort order: 'ascending'
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV gps_bend_diag_${CDATE}.nc4 
+  simulated variables: [bendingAngle]
+geovals:
+  filename: !ENV gps_bend_geoval_${CDATE}.nc4
+obs operator:
+  name: GnssroBndNBAM
+  obs options:
+    use_compress: 1
+    sr_steps: 2
+    vertlayer: full
+    super_ref_qc: NBAM
+obs filters:
+#1. gpstop
+- filter: Domain Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/impactHeightRO
+    minvalue: 0
+    maxvalue: 55000.1
+  action:
+    name: reject
+#2. commgpstop
+- filter: Bounds Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/satelliteIdentifier
+    is_in: 265,266,267,268,269
+  test variables:
+  - name: MetaData/impactHeightRO
+  maxvalue: 45000.1
+  action:
+    name: reject
+#3. metop below 8 km
+- filter: Bounds Check
+  filter variables:
+  - name: bendingAngle
+  where:
+  - variable:
+      name: MetaData/satelliteIdentifier
+    is_in: 3-5
+  test variables:
+  - name: MetaData/impactHeightRO
+  minvalue: 8000.1
+  action:
+    name: reject
+#4. assign obs error 
+- filter: ROobserror
+  filter variables:
+  - name: bendingAngle
+  errmodel: NBAM
+#5. RONBAM cut off check
+- filter: Background Check RONBAM
+  filter variables:
+  - name: bendingAngle
+#6. Obs error inflate
+- filter: Background Check RONBAM
+  filter variables:
+  - name: bendingAngle
+  action:
+    name: RONBAMErrInflate
+#7. Background check
+#- filter: Background Check
+#  filter variables:
+#  - name: bendingAngle
+#  threshold: 10
+#  action:
+#    name: reject
+
+passedBenchmark: 393457
diff --git a/parm/soca/obs/config/icec_amsr2_north.yaml b/parm/soca/obs/config/icec_amsr2_north.yaml
@@ -1,5 +1,5 @@
 obs space:
-  name: icec_ssmis_f17_north
+  name: icec_amsr2_north
   obsdatain:
     engine:
       type: H5File

diff --git a/parm/soca/obs/config/icec_amsr2_south.yaml b/parm/soca/obs/config/icec_amsr2_south.yaml
@@ -1,5 +1,5 @@
 obs space:
-  name: icec_ssmis_f17_south
+  name: icec_amsr2_south
   obsdatain:
     engine:
       type: H5File

diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# exglobal_prep_ocean_obs.py
+# Pepares observations for marine DA
+import os
+from wxflow import YAMLFile
+import prep_marine_obs
+import logging
+
+OBS_YAML = os.getenv('OBS_YAML')
+
+data = YAMLFile(OBS_YAML)
+print(data)
+
+for observer in data['observers']:
+    obs_source_name = observer['obs space']['name']
+    logging.info(f"obs_source_name: {obs_source_name}")
+    prep_marine_obs.obs_fetch(obs_source_name)
diff --git a/ush/soca/obs_list_test.yaml b/ush/soca/obs_list_test.yaml
@@ -0,0 +1,66 @@
+observers:
+- !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_6a_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_all.yaml
+- !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_coperl4.yaml
+- !INC ${OBS_YAML_DIR}/adt_j2_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml
+- !INC ${OBS_YAML_DIR}/bufr_sfcshipsu.yaml
+- !INC ${OBS_YAML_DIR}/bufr_sfcships.yaml
+- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml
+- !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml
+- !INC ${OBS_YAML_DIR}/icec_emc.yaml
+- !INC ${OBS_YAML_DIR}/icec_nsidc_nh.yaml
+- !INC ${OBS_YAML_DIR}/icec_nsidc_sh.yaml
+- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml
+- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml
+- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml
+- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml
+- !INC ${OBS_YAML_DIR}/icefb_gdr.yaml
+- !INC ${OBS_YAML_DIR}/salt_bufr_dbuoyprof.yaml
+- !INC ${OBS_YAML_DIR}/salt_bufr_mbuoybprof.yaml
+- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc_godas.yaml
+- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/salt_profile_wod.yaml
+- !INC ${OBS_YAML_DIR}/sss_smap_jpl.yaml
+- !INC ${OBS_YAML_DIR}/sss_smap.yaml
+- !INC ${OBS_YAML_DIR}/sss_smos_esa.yaml
+- !INC ${OBS_YAML_DIR}/sss_smos.yaml
+- !INC ${OBS_YAML_DIR}/sss_trak_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/sst_amsr2_l3u.yaml
+- !INC ${OBS_YAML_DIR}/sst_drifter.yaml
+- !INC ${OBS_YAML_DIR}/sst_gmi_l3u.yaml
+- !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_noaa18_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml
+- !INC ${OBS_YAML_DIR}/sst_ostia.yaml
+- !INC ${OBS_YAML_DIR}/sst_ship_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/sst_trak_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/sst_viirs_n20_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml
+- !INC ${OBS_YAML_DIR}/sst_windsat_l3u.yaml
+- !INC ${OBS_YAML_DIR}/temp_bufr_dbuoyprof.yaml
+- !INC ${OBS_YAML_DIR}/temp_bufr_mbuoybprof.yaml
+- !INC ${OBS_YAML_DIR}/temp_profile_fnmoc_godas.yaml
+- !INC ${OBS_YAML_DIR}/temp_profile_fnmoc.yaml
+- !INC ${OBS_YAML_DIR}/temp_profile_wod.yaml
+- !INC ${OBS_YAML_DIR}/insitu_argo_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_glider_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_bathy_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_tesac_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_xbtctd_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_dbuoy_profile.yaml,
+- !INC ${OBS_YAML_DIR}/insitu_mbuoy_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_trkob_surface.yaml
+- !INC ${OBS_YAML_DIR}/insitu_altkob_surface.yaml
+- !INC ${OBS_YAML_DIR}/insitu_ships_surface.yaml
+- !INC ${OBS_YAML_DIR}/insitu shipsu_surface.yaml
+- !INC ${OBS_YAML_DIR}/insitu_marinemammal_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_saldrn_profile.yaml
+- !INC ${OBS_YAML_DIR}/insitu_axbt_profile.yaml`
diff --git a/ush/soca/prep_marine_obs.py b/ush/soca/prep_marine_obs.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+
+from wxflow import FileHandler
+import os
+import fnmatch
+
+
+DMPDIR = os.getenv('DMPDIR')
+cyc = os.getenv('cyc')
+PDY = os.getenv('PDY')
+CDUMP = os.getenv('CDUMP')
+COMIN_OBS = os.getenv('COMIN_OBS')
+
+cycdir = os.path.join(DMPDIR, CDUMP + '.' + str(PDY), str(cyc))
+
+# TODO: this looks good for a yaml
+obs_dict = {
+    #                  '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc'
+    'ABI_G16': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G16-ACSPO_V2.70-v02.0-fv01.0.nc '),
+
+    #                  '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc'
+    'ABI_G17': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G17-ACSPO_V2.71-v02.0-fv01.0.nc '),
+
+    #                  '20210630210000-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc'
+    'ABI_G18': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-ABI_G18-ACSPO_V2.71-v02.0-fv01.0.nc '),
+
+    #                  '20210630210000-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc'
+    'AHI_H09': ('sst', '??????????????-STAR-L3C_GHRSST-SSTsubskin-AHI_H09-ACSPO_V2.70-v02.0-fv01.0.nc '),
+
+    #                   'rads_adt_3b_2021182.nc'
+    'adt_3b_egm2008': ('ADT', 'rads_adt_3b_???????.nc'),
+
+    #                   'rads_adt_c2_2021182.nc'
+    'adt_c2_egm2008': ('ADT', 'rads_adt_c2_???????.nc'),
+
+    #                   'rads_adt_sa_2021182.nc'
+    'adt_sa_egm2008': ('ADT', 'rads_adt_sa_???????.nc'),
+
+    #                            'AMSR2-SEAICE-NH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc'
+    'icec_amsr2_north': ('icec', 'AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'),
+
+    #                            'AMSR2-SEAICE-SH_v2r2_GW1_s202107011426180_e202107011605170_c202107011642250.nc'
+    'icec_amsr2_south': ('icec', 'AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc'),
+
+    #                   'SM_OPER_MIR_OSUDP2_20210701T093946_20210701T103256_700_001_1.nc'
+    'sss_smos': ('SSS', 'SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc'),
+
+    #                   'SMAP_L2B_SSS_NRT_34268_A_20210701T153914.h5'
+    'sss_smap': ('SSS', 'SMAP_L2B_SSS_NRT_?????_[AD]_????????T??????.h5'),
+
+    #                               '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc'
+    'sst_metopa_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MA-ACSPO_V2.70-v02.0-fv01.0.nc'),
+
+    #                               '20210701145000-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc'
+    'sst_metopb_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.70-v02.0-fv01.0.nc'),
+
+    #                     '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc'
+    'sst_viirs_n20_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_N20-ACSPO_V2.61-v02.0-fv01.0.nc'),
+
+    #                     '20210701150000-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc'
+    'sst_viirs_npp_l3u_so025': ('sst', '??????????????-OSPO-L3U_GHRSST-SSTsubskin-VIIRS_NPP-ACSPO_V2.61-v02.0-fv01.0.nc'),
+
+}
+
+
+def obs_fetch(obs_source_name):
+
+    try:
+        obs_source = obs_dict[obs_source_name]
+    except KeyError:
+        print(f'WARNING: no obs source {obs_source_name} defined, skipping')
+        return
+
+    subdir = obs_source[0]
+    filepattern = obs_source[1]
+
+    datadir = os.path.join(cycdir, subdir)
+    # TODO: check the existence of this
+    print('datadir:', datadir)
+    matching_files = []
+
+    for root, _, files in os.walk(datadir):
+        for filename in fnmatch.filter(files, filepattern):
+            matching_files.append((root, filename))
+
+    obs_cpy = []
+    for obs_src in matching_files:
+        obs_path = os.path.join(obs_src[0], obs_src[1])
+        obs_dst = os.path.join(COMIN_OBS, obs_src[1])
+        obs_cpy.append([obs_path, obs_dst])
+
+    print(obs_cpy)
+
+    FileHandler({'copy': obs_cpy}).sync()