diff --git a/build.sh b/build.sh index 1d80abba8..e1bd96688 100755 --- a/build.sh +++ b/build.sh @@ -70,7 +70,7 @@ while getopts "p:t:c:hvdfa" opt; do done case ${BUILD_TARGET} in - hera | orion) + hera | orion | hercules) echo "Building GDASApp on $BUILD_TARGET" source $dir_root/ush/module-setup.sh module use $dir_root/modulefiles diff --git a/modulefiles/GDAS/hercules.lua b/modulefiles/GDAS/hercules.lua new file mode 100644 index 000000000..e989af14a --- /dev/null +++ b/modulefiles/GDAS/hercules.lua @@ -0,0 +1,98 @@ +help([[ +Load environment for running the GDAS application with Intel compilers and MPI. +]]) + +local pkgName = myModuleName() +local pkgVersion = myModuleVersion() +local pkgNameVer = myModuleFullName() + +prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core') +prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack') + +-- below two lines get us access to the spack-stack modules +load("stack-intel/2021.9.0") +load("stack-intel-oneapi-mpi/2021.9.0") +--load("stack-python/3.10.8") +-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed +load("cmake/3.23.1") +load("curl/8.1.2") +load("zlib/1.2.13") +load("git/2.40.0") +--load("pkg-config/0.27.1") +load("hdf5/1.14.0") +load("parallel-netcdf/1.12.2") +load("netcdf-c/4.9.2") +load("nccmp/1.9.0.1") +load("netcdf-fortran/4.6.0") +load("nco/5.0.6") +load("parallelio/2.5.10") +load("wget/1.21.1") +load("boost/1.78") +load("bufr/12.0.1") +load("git-lfs/3.1.2") +load("ecbuild/3.7.2") +load("openjpeg/2.4.0") +load("eccodes/2.27.0") +load("eigen/3.4.0") +load("openblas/0.3.19") +load("eckit/1.24.4") +load("fftw/3.3.10") +load("fckit/0.11.0") +load("fiat/1.2.0") +load("ectrans/1.2.0") +load("atlas/0.35.0") +load("sp/2.3.3") +load("gsl-lite/0.37.0") +load("libjpeg/2.1.0") +load("krb5/1.20.1") +load("libtirpc/1.2.6") +load("hdf/4.2.15") +load("jedi-cmake/1.4.0") +load("libpng/1.6.37") +load("libxt/1.1.5") +load("libxmu/1.1.4") +load("libxpm/3.5.12") +load("libxaw/1.0.13") +load("udunits/2.2.28") +load("ncview/2.1.8") +load("netcdf-cxx4/4.3.1") +load("py-pybind11/2.8.1") +--load("crtm/v2.4_jedi") +load("contrib/0.1") +load("noaatools/3.1") +load("rocoto/1.3.5") + +load("hpc/1.2.0") +unload("python/3.10.8") +unload("python/3.9.2") +load("miniconda3/4.6.14") +load("gdasapp/1.0.0") + +load("boost/1.78.0") +load("bufr/12.0.1") +load("fckit/0.11.0") +load("atlas/0.35.0") +load("py-pybind11/2.8.1") + +-- below is a hack because of cmake finding the wrong python... +setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/") + +setenv("CC","mpiicc") +setenv("FC","mpiifort") +setenv("CXX","mpiicpc") +local mpiexec = '/opt/slurm/bin/srun' +local mpinproc = '-n' +setenv('MPIEXEC_EXEC', mpiexec) +setenv('MPIEXEC_NPROC', mpinproc) + +setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml') +setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0") +setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data") +prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin") + +execute{cmd="ulimit -s unlimited",modeA={"load"}} + +whatis("Name: ".. pkgName) +whatis("Version: ".. pkgVersion) +whatis("Category: GDASApp") +whatis("Description: Load all libraries needed for GDASApp") diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml index 55fc33ebf..da2540852 100644 --- a/parm/land/letkfoi/letkfoi.yaml +++ b/parm/land/letkfoi/letkfoi.yaml @@ -22,6 +22,7 @@ geometry: time window: begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}' length: $(LAND_WINDOW_LENGTH) + bound to include: begin background: datetime: '{{ current_cycle | to_isotime }}' diff --git a/parm/soca/obs/config/adt_all.yaml b/parm/soca/obs/config/adt_rads_all.yaml similarity index 88% rename from parm/soca/obs/config/adt_all.yaml rename to parm/soca/obs/config/adt_rads_all.yaml index b7c6697dc..fe41a50da 100644 --- a/parm/soca/obs/config/adt_all.yaml +++ b/parm/soca/obs/config/adt_rads_all.yaml @@ -1,13 +1,13 @@ obs space: - name: adt_all + name: adt_rads_all obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_all.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_rads_all.${PDY}${cyc}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/adt_all.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/diags/adt_rads_all.${PDY}${cyc}.nc4 simulated variables: [absoluteDynamicTopography] io pool: max pool size: 1 diff --git a/parm/soca/obs/config/bufr_sfcships.yaml b/parm/soca/obs/config/bufr/bufr_sfcships.yaml similarity index 100% rename from parm/soca/obs/config/bufr_sfcships.yaml rename to parm/soca/obs/config/bufr/bufr_sfcships.yaml diff --git a/parm/soca/obs/config/bufr_sfcshipsu.yaml b/parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml similarity index 100% rename from parm/soca/obs/config/bufr_sfcshipsu.yaml rename to parm/soca/obs/config/bufr/bufr_sfcshipsu.yaml diff --git a/parm/soca/obs/config/obs_list.yaml b/parm/soca/obs/config/obs_list.yaml deleted file mode 120000 index 372f2edc2..000000000 --- a/parm/soca/obs/config/obs_list.yaml +++ /dev/null @@ -1 +0,0 @@ -../obs_list.yaml \ No newline at end of file diff --git a/parm/soca/obs/config/adt_3a_egm2008.yaml b/parm/soca/obs/config/old/adt_3a_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_3a_egm2008.yaml rename to parm/soca/obs/config/old/adt_3a_egm2008.yaml diff --git a/parm/soca/obs/config/adt_3b_egm2008.yaml b/parm/soca/obs/config/old/adt_3b_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_3b_egm2008.yaml rename to parm/soca/obs/config/old/adt_3b_egm2008.yaml diff --git a/parm/soca/obs/config/adt_6a_egm2008.yaml b/parm/soca/obs/config/old/adt_6a_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_6a_egm2008.yaml rename to parm/soca/obs/config/old/adt_6a_egm2008.yaml diff --git a/parm/soca/obs/config/adt_c2_egm2008.yaml b/parm/soca/obs/config/old/adt_c2_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_c2_egm2008.yaml rename to parm/soca/obs/config/old/adt_c2_egm2008.yaml diff --git a/parm/soca/obs/config/adt_coperl4.yaml b/parm/soca/obs/config/old/adt_coperl4.yaml similarity index 100% rename from parm/soca/obs/config/adt_coperl4.yaml rename to parm/soca/obs/config/old/adt_coperl4.yaml diff --git a/parm/soca/obs/config/adt_j2.yaml b/parm/soca/obs/config/old/adt_j2.yaml similarity index 100% rename from parm/soca/obs/config/adt_j2.yaml rename to parm/soca/obs/config/old/adt_j2.yaml diff --git a/parm/soca/obs/config/adt_j2_egm2008.yaml b/parm/soca/obs/config/old/adt_j2_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_j2_egm2008.yaml rename to parm/soca/obs/config/old/adt_j2_egm2008.yaml diff --git a/parm/soca/obs/config/adt_j3.yaml b/parm/soca/obs/config/old/adt_j3.yaml similarity index 100% rename from parm/soca/obs/config/adt_j3.yaml rename to parm/soca/obs/config/old/adt_j3.yaml diff --git a/parm/soca/obs/config/adt_j3_egm2008.yaml b/parm/soca/obs/config/old/adt_j3_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_j3_egm2008.yaml rename to parm/soca/obs/config/old/adt_j3_egm2008.yaml diff --git a/parm/soca/obs/config/adt_sa_egm2008.yaml b/parm/soca/obs/config/old/adt_sa_egm2008.yaml similarity index 100% rename from parm/soca/obs/config/adt_sa_egm2008.yaml rename to parm/soca/obs/config/old/adt_sa_egm2008.yaml diff --git a/parm/soca/obs/config/icec_emc.yaml b/parm/soca/obs/config/old/icec_emc.yaml similarity index 100% rename from parm/soca/obs/config/icec_emc.yaml rename to parm/soca/obs/config/old/icec_emc.yaml diff --git a/parm/soca/obs/config/icec_nsidc_nh.yaml b/parm/soca/obs/config/old/icec_nsidc_nh.yaml similarity index 100% rename from parm/soca/obs/config/icec_nsidc_nh.yaml rename to parm/soca/obs/config/old/icec_nsidc_nh.yaml diff --git a/parm/soca/obs/config/icec_nsidc_sh.yaml b/parm/soca/obs/config/old/icec_nsidc_sh.yaml similarity index 100% rename from parm/soca/obs/config/icec_nsidc_sh.yaml rename to parm/soca/obs/config/old/icec_nsidc_sh.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f17_north.yaml b/parm/soca/obs/config/old/icec_ssmis_f17_north.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f17_north.yaml rename to parm/soca/obs/config/old/icec_ssmis_f17_north.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f17_south.yaml b/parm/soca/obs/config/old/icec_ssmis_f17_south.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f17_south.yaml rename to parm/soca/obs/config/old/icec_ssmis_f17_south.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_north.yaml b/parm/soca/obs/config/old/icec_ssmis_f18_north.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f18_north.yaml rename to parm/soca/obs/config/old/icec_ssmis_f18_north.yaml diff --git a/parm/soca/obs/config/icec_ssmis_f18_south.yaml b/parm/soca/obs/config/old/icec_ssmis_f18_south.yaml similarity index 100% rename from parm/soca/obs/config/icec_ssmis_f18_south.yaml rename to parm/soca/obs/config/old/icec_ssmis_f18_south.yaml diff --git a/parm/soca/obs/config/icefb_gdr.yaml b/parm/soca/obs/config/old/icefb_gdr.yaml similarity index 100% rename from parm/soca/obs/config/icefb_gdr.yaml rename to parm/soca/obs/config/old/icefb_gdr.yaml diff --git a/parm/soca/obs/config/oc_aqua.yaml b/parm/soca/obs/config/old/oc_aqua.yaml similarity index 100% rename from parm/soca/obs/config/oc_aqua.yaml rename to parm/soca/obs/config/old/oc_aqua.yaml diff --git a/parm/soca/obs/config/oc_dineof3.yaml b/parm/soca/obs/config/old/oc_dineof3.yaml similarity index 100% rename from parm/soca/obs/config/oc_dineof3.yaml rename to parm/soca/obs/config/old/oc_dineof3.yaml diff --git a/parm/soca/obs/config/oc_noaa20.yaml b/parm/soca/obs/config/old/oc_noaa20.yaml similarity index 100% rename from parm/soca/obs/config/oc_noaa20.yaml rename to parm/soca/obs/config/old/oc_noaa20.yaml diff --git a/parm/soca/obs/config/oc_snpp.yaml b/parm/soca/obs/config/old/oc_snpp.yaml similarity index 100% rename from parm/soca/obs/config/oc_snpp.yaml rename to parm/soca/obs/config/old/oc_snpp.yaml diff --git a/parm/soca/obs/config/salt_bufr_dbuoyprof.yaml b/parm/soca/obs/config/old/salt_bufr_dbuoyprof.yaml similarity index 100% rename from parm/soca/obs/config/salt_bufr_dbuoyprof.yaml rename to parm/soca/obs/config/old/salt_bufr_dbuoyprof.yaml diff --git a/parm/soca/obs/config/salt_bufr_mbuoybprof.yaml b/parm/soca/obs/config/old/salt_bufr_mbuoybprof.yaml similarity index 100% rename from parm/soca/obs/config/salt_bufr_mbuoybprof.yaml rename to parm/soca/obs/config/old/salt_bufr_mbuoybprof.yaml diff --git a/parm/soca/obs/config/salt_profile_argo.yaml b/parm/soca/obs/config/old/salt_profile_argo.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_argo.yaml rename to parm/soca/obs/config/old/salt_profile_argo.yaml diff --git a/parm/soca/obs/config/salt_profile_fnmoc.yaml b/parm/soca/obs/config/old/salt_profile_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_fnmoc.yaml rename to parm/soca/obs/config/old/salt_profile_fnmoc.yaml diff --git a/parm/soca/obs/config/salt_profile_fnmoc_godas.yaml b/parm/soca/obs/config/old/salt_profile_fnmoc_godas.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_fnmoc_godas.yaml rename to parm/soca/obs/config/old/salt_profile_fnmoc_godas.yaml diff --git a/parm/soca/obs/config/salt_profile_glider.yaml b/parm/soca/obs/config/old/salt_profile_glider.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_glider.yaml rename to parm/soca/obs/config/old/salt_profile_glider.yaml diff --git a/parm/soca/obs/config/salt_profile_tesac.yaml b/parm/soca/obs/config/old/salt_profile_tesac.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_tesac.yaml rename to parm/soca/obs/config/old/salt_profile_tesac.yaml diff --git a/parm/soca/obs/config/salt_profile_wod.yaml b/parm/soca/obs/config/old/salt_profile_wod.yaml similarity index 100% rename from parm/soca/obs/config/salt_profile_wod.yaml rename to parm/soca/obs/config/old/salt_profile_wod.yaml diff --git a/parm/soca/obs/config/sss_smap.yaml b/parm/soca/obs/config/old/sss_smap.yaml similarity index 100% rename from parm/soca/obs/config/sss_smap.yaml rename to parm/soca/obs/config/old/sss_smap.yaml diff --git a/parm/soca/obs/config/sss_smap_jpl.yaml b/parm/soca/obs/config/old/sss_smap_jpl.yaml similarity index 100% rename from parm/soca/obs/config/sss_smap_jpl.yaml rename to parm/soca/obs/config/old/sss_smap_jpl.yaml diff --git a/parm/soca/obs/config/sss_smos.yaml b/parm/soca/obs/config/old/sss_smos.yaml similarity index 100% rename from parm/soca/obs/config/sss_smos.yaml rename to parm/soca/obs/config/old/sss_smos.yaml diff --git a/parm/soca/obs/config/sss_smos_esa.yaml b/parm/soca/obs/config/old/sss_smos_esa.yaml similarity index 100% rename from parm/soca/obs/config/sss_smos_esa.yaml rename to parm/soca/obs/config/old/sss_smos_esa.yaml diff --git a/parm/soca/obs/config/sss_trak_fnmoc.yaml b/parm/soca/obs/config/old/sss_trak_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sss_trak_fnmoc.yaml rename to parm/soca/obs/config/old/sss_trak_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_amsr2_l3u.yaml b/parm/soca/obs/config/old/sst_amsr2_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_amsr2_l3u.yaml rename to parm/soca/obs/config/old/sst_amsr2_l3u.yaml diff --git a/parm/soca/obs/config/sst_drifter.yaml b/parm/soca/obs/config/old/sst_drifter.yaml similarity index 100% rename from parm/soca/obs/config/sst_drifter.yaml rename to parm/soca/obs/config/old/sst_drifter.yaml diff --git a/parm/soca/obs/config/sst_gmi_l3u.yaml b/parm/soca/obs/config/old/sst_gmi_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_gmi_l3u.yaml rename to parm/soca/obs/config/old/sst_gmi_l3u.yaml diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/old/sst_metopa_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_metopa_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_metopa_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/old/sst_metopc_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_metopc_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_metopc_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/old/sst_noaa18_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa18_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_noaa18_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u.yaml b/parm/soca/obs/config/old/sst_noaa19_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa19_l3u.yaml rename to parm/soca/obs/config/old/sst_noaa19_l3u.yaml diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/old/sst_noaa19_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_noaa19_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_noaa19_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_ostia.yaml b/parm/soca/obs/config/old/sst_ostia.yaml similarity index 100% rename from parm/soca/obs/config/sst_ostia.yaml rename to parm/soca/obs/config/old/sst_ostia.yaml diff --git a/parm/soca/obs/config/sst_ship_fnmoc.yaml b/parm/soca/obs/config/old/sst_ship_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sst_ship_fnmoc.yaml rename to parm/soca/obs/config/old/sst_ship_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_trak_fnmoc.yaml b/parm/soca/obs/config/old/sst_trak_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/sst_trak_fnmoc.yaml rename to parm/soca/obs/config/old/sst_trak_fnmoc.yaml diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/old/sst_viirs_n20_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_viirs_n20_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/old/sst_viirs_npp_l3u_so025.yaml similarity index 100% rename from parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml rename to parm/soca/obs/config/old/sst_viirs_npp_l3u_so025.yaml diff --git a/parm/soca/obs/config/sst_windsat_l3u.yaml b/parm/soca/obs/config/old/sst_windsat_l3u.yaml similarity index 100% rename from parm/soca/obs/config/sst_windsat_l3u.yaml rename to parm/soca/obs/config/old/sst_windsat_l3u.yaml diff --git a/parm/soca/obs/config/temp_bufr_dbuoyprof.yaml b/parm/soca/obs/config/old/temp_bufr_dbuoyprof.yaml similarity index 100% rename from parm/soca/obs/config/temp_bufr_dbuoyprof.yaml rename to parm/soca/obs/config/old/temp_bufr_dbuoyprof.yaml diff --git a/parm/soca/obs/config/temp_bufr_mbuoybprof.yaml b/parm/soca/obs/config/old/temp_bufr_mbuoybprof.yaml similarity index 100% rename from parm/soca/obs/config/temp_bufr_mbuoybprof.yaml rename to parm/soca/obs/config/old/temp_bufr_mbuoybprof.yaml diff --git a/parm/soca/obs/config/temp_profile_argo.yaml b/parm/soca/obs/config/old/temp_profile_argo.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_argo.yaml rename to parm/soca/obs/config/old/temp_profile_argo.yaml diff --git a/parm/soca/obs/config/temp_profile_bathy.yaml b/parm/soca/obs/config/old/temp_profile_bathy.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_bathy.yaml rename to parm/soca/obs/config/old/temp_profile_bathy.yaml diff --git a/parm/soca/obs/config/temp_profile_fnmoc.yaml b/parm/soca/obs/config/old/temp_profile_fnmoc.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_fnmoc.yaml rename to parm/soca/obs/config/old/temp_profile_fnmoc.yaml diff --git a/parm/soca/obs/config/temp_profile_fnmoc_godas.yaml b/parm/soca/obs/config/old/temp_profile_fnmoc_godas.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_fnmoc_godas.yaml rename to parm/soca/obs/config/old/temp_profile_fnmoc_godas.yaml diff --git a/parm/soca/obs/config/temp_profile_glider.yaml b/parm/soca/obs/config/old/temp_profile_glider.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_glider.yaml rename to parm/soca/obs/config/old/temp_profile_glider.yaml diff --git a/parm/soca/obs/config/temp_profile_tesac.yaml b/parm/soca/obs/config/old/temp_profile_tesac.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_tesac.yaml rename to parm/soca/obs/config/old/temp_profile_tesac.yaml diff --git a/parm/soca/obs/config/temp_profile_wod.yaml b/parm/soca/obs/config/old/temp_profile_wod.yaml similarity index 100% rename from parm/soca/obs/config/temp_profile_wod.yaml rename to parm/soca/obs/config/old/temp_profile_wod.yaml diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u.yaml similarity index 84% rename from parm/soca/obs/config/sst_metopb_l3u_so025.yaml rename to parm/soca/obs/config/sst_metopb_l3u.yaml index 8865929c1..6b2fc9ac5 100644 --- a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml +++ b/parm/soca/obs/config/sst_metopb_l3u.yaml @@ -1,13 +1,13 @@ obs space: - name: sst_metopb_l3u_so025 + name: sst_metopb_l3u obsdatain: engine: type: H5File - obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u_so025.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u.${PDY}${cyc}.nc4 obsdataout: engine: type: H5File - obsfile: !ENV ${DATA}/diags/sst_metopb_l3u_so025.${PDY}${cyc}.nc4 + obsfile: !ENV ${DATA}/diags/sst_metopb_l3u.${PDY}${cyc}.nc4 simulated variables: [seaSurfaceTemperature] io pool: max pool size: 1 diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml index 89b2659a1..811872f66 100644 --- a/parm/soca/obs/obs_list.yaml +++ b/parm/soca/obs/obs_list.yaml @@ -1,22 +1,5 @@ observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml -- !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml -- !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -#- !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml -#- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml -#- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml -- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_argo.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_bathy.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_glider.yaml -#- !INC ${OBS_YAML_DIR}/temp_profile_tesac.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_argo.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_glider.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_tesac.yaml \ No newline at end of file +- !INC ${OBS_YAML_DIR}/adt_rads_all.yaml +- !INC ${OBS_YAML_DIR}/sst_metopb_l3u.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml +- !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml diff --git a/parm/soca/obs/obs_list_small.yaml b/parm/soca/obs/obs_list_small.yaml deleted file mode 100644 index 4ff437200..000000000 --- a/parm/soca/obs/obs_list_small.yaml +++ /dev/null @@ -1,9 +0,0 @@ -observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3.yaml -- !INC ${OBS_YAML_DIR}/adt_j2.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml -- !INC ${OBS_YAML_DIR}/sss_smap.yaml -- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml -- !INC ${OBS_YAML_DIR}/icec_emc.yaml diff --git a/parm/soca/obsproc/obs_list_obsproc.yaml b/parm/soca/obsproc/obs_list_obsproc.yaml deleted file mode 100644 index 7a884f9ea..000000000 --- a/parm/soca/obsproc/obs_list_obsproc.yaml +++ /dev/null @@ -1,12 +0,0 @@ -observers: -- !INC ${OBS_YAML_DIR}/adt_all.yaml -- !INC ${OBS_YAML_DIR}/adt_j3.yaml -- !INC ${OBS_YAML_DIR}/adt_j2.yaml -#- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml -- !INC ${OBS_YAML_DIR}/sss_smap.yaml -- !INC ${OBS_YAML_DIR}/sss_smos.yaml -#- !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml -- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml -- !INC ${OBS_YAML_DIR}/icec_emc.yaml -- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml -- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml diff --git a/parm/soca/obsproc/obsproc_config.yaml b/parm/soca/obsproc/obsproc_config.yaml index 60cb0df66..5102ac9a3 100644 --- a/parm/soca/obsproc/obsproc_config.yaml +++ b/parm/soca/obsproc/obsproc_config.yaml @@ -12,14 +12,30 @@ observations: output file: sss_smos.ioda.nc obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc - obs space: - name: adt_3b_egm2008 + name: adt_rads_all obsproc subdir: ADT - obsproc regex: rads_adt_3b_???????.nc + obsproc regex: rads_adt_??_???????.nc provider: RADS - output file: adt_all.nc4 + output file: adt_rads_all.nc4 - obs space: name: icec_amsr2_north provider: AMSR2 obsproc subdir: icec - output file: icec_amsr2_north_1.ioda.nc + output file: icec_amsr2_north.ioda.nc obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: icec_amsr2_south + provider: AMSR2 + obsproc subdir: icec + output file: icec_amsr2_south.ioda.nc + obsproc regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc +- obs space: + name: sst_metopb_l3u + provider: GHRSST + obsproc subdir: 'sst' + output file: sst_metopb_l3u.ioda.nc + obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' + bounds: + units: C + min: -3.0 + max: 50.0 diff --git a/parm/soca/obsproc/obsproc_config_test.yaml b/parm/soca/obsproc/obsproc_config_test.yaml deleted file mode 100644 index 288aee4f4..000000000 --- a/parm/soca/obsproc/obsproc_config_test.yaml +++ /dev/null @@ -1,35 +0,0 @@ -observations: -- obs space: - name: sss_smap - obsproc subdir: 'SSS' - obsproc regex: sss_smap_?.nc4 - provider: SMAP - output file: sss_smap.ioda.nc -- obs space: - name: sss_smos - provider: SMOS - obsproc subdir: 'SSS' - output file: sss_smos.ioda.nc - obsproc regex: sss_smos_?.nc4 -- obs space: - name: adt_3b_egm2008 - obsproc subdir: 'ADT' - obsproc regex: rads_adt_??_???????.nc4 - provider: RADS - output file: adt_all.nc4 -- obs space: - name: icec_amsr2_north - provider: AMSR2 - obsproc subdir: 'ice' - output file: icec_amsr2_north.ioda.nc - obsproc regex: icec_amsr2_north_?.nc4 -- obs space: - name: sst_metopb_l3u_so025 - provider: GHRSST - obsproc subdir: 'sst' - output file: sst_metopb_l3u_so025.ioda.nc - obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc' - bounds: - units: C - min: -3.0 - max: 50.0 diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py index 4af8d0189..d2cda4b1b 100755 --- a/scripts/exgdas_global_marine_analysis_prep.py +++ b/scripts/exgdas_global_marine_analysis_prep.py @@ -225,7 +225,7 @@ def find_clim_ens(input_date): logging.info(f"---------------- Setup runtime environement") -comin_obs = os.getenv('COMIN_OBS') # R2D2 DB for now +comin_obs = os.getenv('COMIN_OBS') anl_dir = os.getenv('DATA') staticsoca_dir = os.getenv('SOCA_INPUT_FIX_DIR') if os.getenv('DOHYBVAR') == "YES": @@ -268,11 +268,6 @@ def find_clim_ens(input_date): stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get) stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get) -# concatenate altimeters into one obs space -# TODO (SAMG)temporary, move this into the obs procecing eventually -adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt" -ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True) - # get the list of observations obs_files = [] for ob in stage_cfg['observations']['observers']: diff --git a/test/atm/global-workflow/jjob_ens_final.sh b/test/atm/global-workflow/jjob_ens_final.sh index ae1928a18..f0a0a9117 100755 --- a/test/atm/global-workflow/jjob_ens_final.sh +++ b/test/atm/global-workflow/jjob_ens_final.sh @@ -33,11 +33,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -45,10 +45,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh index 8966f2b69..e46ea90da 100755 --- a/test/atm/global-workflow/jjob_ens_init.sh +++ b/test/atm/global-workflow/jjob_ens_init.sh @@ -38,11 +38,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -69,7 +69,7 @@ dpath=gdas.$PDY/$cyc/obs mkdir -p $COM_OBS flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file done # Link radiance bias correction files @@ -77,7 +77,7 @@ dpath=gdas.$gPDY/$gcyc/analysis/atmos mkdir -p $COM_ATMOS_ANALYSIS_PREV flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file done # Link member atmospheric background on tiles and atmf006 @@ -104,10 +104,10 @@ for imem in $(seq 1 $NMEM_ENS); do done # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_ens_run.sh b/test/atm/global-workflow/jjob_ens_run.sh index 597afcd43..89bff09fe 100755 --- a/test/atm/global-workflow/jjob_ens_run.sh +++ b/test/atm/global-workflow/jjob_ens_run.sh @@ -36,11 +36,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -48,10 +48,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN -elif [ $machine = 'ORION' ]; then - sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --partition=orion --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN else ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN fi diff --git a/test/atm/global-workflow/jjob_var_final.sh b/test/atm/global-workflow/jjob_var_final.sh index f9b64afbe..01b3c555f 100755 --- a/test/atm/global-workflow/jjob_var_final.sh +++ b/test/atm/global-workflow/jjob_var_final.sh @@ -33,11 +33,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -45,10 +45,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_FINALIZE fi diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh index 0d88c2174..6d304e0a8 100755 --- a/test/atm/global-workflow/jjob_var_init.sh +++ b/test/atm/global-workflow/jjob_var_init.sh @@ -38,11 +38,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -71,7 +71,7 @@ dpath=gdas.$PDY/$cyc/obs mkdir -p $COM_OBS flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file done # Link radiance bias correction files @@ -79,7 +79,7 @@ dpath=gdas.$gPDY/$gcyc/analysis/atmos mkdir -p $COM_ATMOS_ANALYSIS_PREV flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file done # Link atmospheric background on gaussian grid @@ -87,7 +87,7 @@ dpath=gdas.$gPDY/$gcyc/model_data/atmos/history mkdir -p $COM_ATMOS_HISTORY_PREV flist="atmf006.nc" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/${gprefix}.${file} done # Link atmospheric bacgkround on tiles @@ -96,7 +96,7 @@ COM_ATMOS_RESTART_PREV_DIRNAME=$(dirname $COM_ATMOS_RESTART_PREV) mkdir -p $COM_ATMOS_RESTART_PREV_DIRNAME flist="restart" for file in $flist; do - ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/ + ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/$file done @@ -125,10 +125,10 @@ done # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_INITIALIZE fi diff --git a/test/atm/global-workflow/jjob_var_run.sh b/test/atm/global-workflow/jjob_var_run.sh index a78b1e10e..e13aa2daa 100755 --- a/test/atm/global-workflow/jjob_var_run.sh +++ b/test/atm/global-workflow/jjob_var_run.sh @@ -36,11 +36,11 @@ export PYTHONPATH machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"') # Set NETCDF and UTILROOT variables (used in config.base) -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then NETCDF=$( which ncdump ) export NETCDF export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2" -elif [ $machine = 'ORION' ]; then +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) export NETCDF @@ -48,10 +48,10 @@ elif [ $machine = 'ORION' ]; then fi # Execute j-job -if [ $machine = 'HERA' ]; then +if [[ $machine = 'HERA' ]]; then + sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN +elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN -elif [ $machine = 'ORION' ]; then - sbatch --ntasks=6 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN else ${HOMEgfs}/jobs/JGLOBAL_ATM_ANALYSIS_RUN fi diff --git a/test/soca/gw/CMakeLists.txt b/test/soca/gw/CMakeLists.txt index 2d7d059d8..5bcb9f71f 100644 --- a/test/soca/gw/CMakeLists.txt +++ b/test/soca/gw/CMakeLists.txt @@ -11,9 +11,14 @@ add_test(NAME test_gdasapp_soca_prep # Identify machine set(MACHINE "container") -IF (IS_DIRECTORY /work2/noaa/da) - set(MACHINE "orion") - set(PARTITION "orion") +IF (IS_DIRECTORY /work2) + IF (IS_DIRECTORY /apps/other) + set(MACHINE "hercules") + set(PARTITION "hercules") + ELSE() + set(MACHINE "orion") + set(PARTITION "orion") + ENDIF() ENDIF() IF (IS_DIRECTORY /scratch2/NCEPDEV/) set(MACHINE "hera") diff --git a/test/soca/gw/prepdata.sh b/test/soca/gw/prepdata.sh index 88cde39bf..74f86b3bc 100755 --- a/test/soca/gw/prepdata.sh +++ b/test/soca/gw/prepdata.sh @@ -16,23 +16,26 @@ cdl2nc4() { project_source_dir=$1 -#cdl2nc4 ADT/rads_adt_3a_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3a_2021181.cdl -#cdl2nc4 ADT/rads_adt_3b_2021181.nc4 ${project_source_dir}/testdata/rads_adt_3b_2021181.cdl -#cdl2nc4 icec/icec_amsr2_north_1.nc4 ${project_source_dir}/testdata/icec_amsr2_north_1.cdl -#cdl2nc4 icec/icec_amsr2_north_2.nc4 ${project_source_dir}/testdata/icec_amsr2_north_2.cdl -#cdl2nc4 icec/icec_amsr2_south_1.nc4 ${project_source_dir}/testdata/icec_amsr2_south_1.cdl -#cdl2nc4 icec/icec_amsr2_south_2.nc4 ${project_source_dir}/testdata/icec_amsr2_south_2.cdl -#cdl2nc4 SSS/sss_smap_1.nc4 ${project_source_dir}/testdata/sss_smap_1.cdl -#cdl2nc4 SSS/sss_smap_2.nc4 ${project_source_dir}/testdata/sss_smap_2.cdl -#cdl2nc4 SSS/sss_smos_1.nc4 ${project_source_dir}/testdata/sss_smos_1.cdl -#cdl2nc4 SSS/sss_smos_2.nc4 ${project_source_dir}/testdata/sss_smos_2.cdl -#cdl2nc4 sst/ghrsst_sst_mb_202107010000.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010000.cdl -#cdl2nc4 sst/ghrsst_sst_mb_202107010100.nc4 ${project_source_dir}/testdata/ghrsst_sst_mb_202107010100.cdl -#cdl2nc4 sst/viirs_aod_1.nc4 ${project_source_dir}/testdata/viirs_aod_1.cdl -#cdl2nc4 sst/viirs_aod_2.nc4 ${project_source_dir}/testdata/viirs_aod_2.cdl - -cdl2nc4 sst/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl -cdl2nc4 sst/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl -cdl2nc4 sst/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc ${project_source_dir}/test/soca/testdata/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl - - +cdldir=${project_source_dir}/test/soca/testdata + +# GHRSST +cdl2nc4 sst/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415114000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl +cdl2nc4 sst/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415132000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl +cdl2nc4 sst/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc \ + ${cdldir}/20180415095000-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.cdl + +# Fake AMSR2 icec +cdl2nc4 icec/AMSR2-SEAICE-NH_v2r2_GW1_s201804150835180_e201804151014170_c201804151052280.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_north_1.cdl +cdl2nc4 icec/AMSR2-SEAICE-NH_v2r2_GW1_s201804151014190_e201804151150170_c201804151309570.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_north_2.cdl +cdl2nc4 icec/AMSR2-SEAICE-SH_v2r2_GW1_s201804150835180_e201804151014170_c201804151052280.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_south_1.cdl +cdl2nc4 icec/AMSR2-SEAICE-SH_v2r2_GW1_s201804151014190_e201804151150170_c201804151309570.nc \ + ${project_source_dir}/test/soca/testdata/icec_amsr2_south_2.cdl + +# Fake RADS ADT +cdl2nc4 ADT/rads_adt_3a_2018105.nc ${project_source_dir}/test/soca/testdata/rads_adt_3a_2018105.cdl +cdl2nc4 ADT/rads_adt_3b_2018105.nc ${project_source_dir}/test/soca/testdata/rads_adt_3b_2018105.cdl diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test index ee42ad7f6..a185cee2b 100644 --- a/test/soca/gw/run_jjobs.yaml.test +++ b/test/soca/gw/run_jjobs.yaml.test @@ -48,14 +48,14 @@ setup_expt config: ocnanal: SOCA_INPUT_FIX_DIR: @HOMEgfs@/sorc/gdas.cd/build/soca_static CASE_ANL: C48 - SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list_small.yaml + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 1 SABER_BLOCKS_YAML: @HOMEgfs@/sorc/gdas.cd/parm/soca/berror/saber_blocks.yaml NICAS_RESOL: 1 NICAS_GRID_SIZE: 150 prepoceanobs: - SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obs_list_obsproc.yaml - OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config_test.yaml + SOCA_OBS_LIST: @HOMEgfs@/sorc/gdas.cd/parm/soca/obs/obs_list.yaml + OBSPROC_CONFIG: @HOMEgfs@/sorc/gdas.cd/parm/soca/obsproc/obsproc_config.yaml DMPDIR: @HOMEgfs@/sorc/gdas.cd/build/test/soca/gw/obsproc job options: diff --git a/test/soca/testdata/icec_amsr2_north_1.cdl b/test/soca/testdata/icec_amsr2_north_1.cdl new file mode 100644 index 000000000..9470a6bdb --- /dev/null +++ b/test/soca/testdata/icec_amsr2_north_1.cdl @@ -0,0 +1,208 @@ +netcdf output { +dimensions: + Number_of_X_Dimension = 3 ; + Number_of_Y_Dimension = 4 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction) - secondary product" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction) - primary product" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction) - based on NT2" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE" ; + :summary = "GCOM Seaice Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = 18.87699f ; + :geospatial_lat_max = 89.9367f ; + :geospatial_lon_min = -179.945f ; + :geospatial_lon_max = 179.945f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T02:02:39Z" ; + :id = "abb20bb6-945f-4f02-a7a5-14f6dcdce4cf" ; + :Metadata_Link = "AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc" ; + :history = "Mon Sep 25 15:10:33 2023: ncks -d Number_of_X_Dimension,400,600,100 -d Number_of_Y_Dimension,400,700,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-NH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc output.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5, GAASP-L1R_v2r2_GW1_s202106292259160_e202106300038160_c202106300112300.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :start_orbit_number = 48513 ; + :end_orbit_number = 48514 ; + :day_night_data_flag = 2 ; + :ascend_descend_data_flag = 2 ; + :time_coverage_start = "2021-06-30T23:41:19.004Z" ; + :time_coverage_end = "2021-07-01T01:20:16.940Z" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 401, 501, 601 ; + + Along_Y_Dimension = 401, 501, 601, 701 ; + + Bootstrap_Ice_Concentration = + 100, 100, 0, + _, 100, 100, + _, 87, 96, + _, 0, 0 ; + + Flags = + 0, 0, 8, + 120, 0, 0, + 120, 0, 0, + 120, 8, 8 ; + + Latency = + 204, 104, 6, + 2000, 299, 3, + 2000, 696, 1, + 2000, 699, 897 ; + + Latitude = + 74.18227, 78.6195, 76.93348, + 78.6195, 86.89752, 82.88853, + 76.93348, 82.88853, 80.42855, + 70.63603, 74.07984, 72.82585 ; + + Longitude = + -135, -168.8672, 148.7663, + -101.1328, -135, 107.9784, + -58.7663, -17.97842, 45, + -35.35196, -7.947196, 23.27735 ; + + NASA_Team_2_Ice_Concentration = + 100, 100, 0, + _, 100, 100, + _, 97, 100, + _, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, + _, 100, 100, + _, 0, 0, + _, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, + _, 0, 0, + _, 10, 4, + _, 0, 0 ; + + QM_Num_Grid_Range_25 = 10331 ; + + QM_Num_Grid_Range_50 = 2560 ; + + QM_Total_Ice = 98806 ; + + QM_Total_Pixels = 1102500 ; + + Range_of_Ice_Concentration = + 1, 3, 0, + _, 0, 3, + _, 13, 4, + _, 0, 0 ; + + Scan_Time = + 2018, 4, 15, 9, 55, 22.60762, + 2018, 4, 15, 9, 35, 23.53783, + 2018, 4, 15, 9, 13, 57.47626, + _, _, _, _, _, _, + 2018, 4, 15, 9, 20, 29.15068, + 2018, 4, 15, 10, 16, 25.96225, + _, _, _, _, _, _, + 2018, 4, 15, 11, 43, 40.417, + 2018, 4, 15, 12, 18, 54.4482, + _, _, _, _, _, _, + 2018, 4, 15, 13, 41, 11.93098, + 2018, 4, 15, 14, 22, 37.06214 ; +} diff --git a/test/soca/testdata/icec_amsr2_north_2.cdl b/test/soca/testdata/icec_amsr2_north_2.cdl new file mode 100644 index 000000000..afb71f57f --- /dev/null +++ b/test/soca/testdata/icec_amsr2_north_2.cdl @@ -0,0 +1,208 @@ +netcdf icec_amsr2_north_2 { +dimensions: + Number_of_X_Dimension = 3 ; + Number_of_Y_Dimension = 4 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction) - secondary product" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction) - primary product" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction) - based on NT2" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE" ; + :summary = "GCOM Seaice Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = 18.87699f ; + :geospatial_lat_max = 89.9367f ; + :geospatial_lon_min = -179.945f ; + :geospatial_lon_max = 179.945f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T03:40:15Z" ; + :id = "84a33a48-3a6d-43f6-bb82-48e9072799a9" ; + :Metadata_Link = "AMSR2-SEAICE-NH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc" ; + :history = "Mon Sep 25 15:41:32 2023: ncks -d Number_of_X_Dimension,400,600,100 -d Number_of_Y_Dimension,400,700,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-NH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc icec_amsr2_north_2.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202107010120180_e202107010259170_c202107010336520.h5, GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :start_orbit_number = 48514 ; + :end_orbit_number = 48515 ; + :day_night_data_flag = 2 ; + :ascend_descend_data_flag = 2 ; + :time_coverage_start = "2021-07-01T01:20:18.440Z" ; + :time_coverage_end = "2021-07-01T02:59:17.876Z" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 401, 501, 601 ; + + Along_Y_Dimension = 401, 501, 601, 701 ; + + Bootstrap_Ice_Concentration = + 100, 95, 0, + _, 100, 100, + _, 87, 95, + _, 0, 0 ; + + Flags = + 0, 0, 8, + 120, 0, 0, + 120, 0, 0, + 120, 8, 4 ; + + Latency = + 303, 203, 6, + 2000, 398, 4, + 2000, 795, 2, + 2000, 798, _ ; + + Latitude = + 74.18227, 78.6195, 76.93348, + 78.6195, 86.89752, 82.88853, + 76.93348, 82.88853, 80.42855, + 70.63603, 74.07984, 72.82585 ; + + Longitude = + -135, -168.8672, 148.7663, + -101.1328, -135, 107.9784, + -58.7663, -17.97842, 45, + -35.35196, -7.947196, 23.27735 ; + + NASA_Team_2_Ice_Concentration = + 100, 100, 0, + _, 100, 97, + _, 97, 100, + _, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 79, 0, + _, 100, 100, + _, 0, 0, + _, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 5, 0, + _, 0, -2, + _, 10, 5, + _, 0, 0 ; + + QM_Num_Grid_Range_25 = 10266 ; + + QM_Num_Grid_Range_50 = 2384 ; + + QM_Total_Ice = 98282 ; + + QM_Total_Pixels = 1102500 ; + + Range_of_Ice_Concentration = + 1, 3, 0, + _, 0, 3, + _, 13, 4, + _, 0, 0 ; + + Scan_Time = + 2018, 4, 15, 9, 55, 22.60762, + 2018, 4, 15, 9, 35, 23.53783, + 2018, 4, 15, 10, 52, 49.41316, + _, _, _, _, _, _, + 2018, 4, 15, 11, 20, 29.15068, + 2018, 4, 15, 12, 54, 38.90287, + _, _, _, _, _, _, + 2018, 4, 15, 13, 43, 40.417, + 2018, 4, 15, 13, 56, 49.3905, + _, _, _, _, _, _, + 2018, 4, 15, 14, 41, 11.93098, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/icec_amsr2_south_1.cdl b/test/soca/testdata/icec_amsr2_south_1.cdl new file mode 100644 index 000000000..7d19a1c74 --- /dev/null +++ b/test/soca/testdata/icec_amsr2_south_1.cdl @@ -0,0 +1,306 @@ +netcdf icec_amsr2_south { +dimensions: + Number_of_X_Dimension = 8 ; + Number_of_Y_Dimension = 8 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction)" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction)" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (Fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction)" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE_SH" ; + :summary = "GCOM Seaice Southern Hemisphere Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = -89.9367f ; + :geospatial_lat_max = -34.6223f ; + :geospatial_lon_min = -179.932f ; + :geospatial_lon_max = 179.932f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T02:02:39Z" ; + :id = "09bd3374-b3bf-46e4-b4bc-f698cccbd717" ; + :Metadata_Link = "AMSR2-SEAICE-SH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc" ; + :history = "Mon Sep 25 15:26:46 2023: ncks -d Number_of_X_Dimension,100,800,100 -d Number_of_Y_Dimension,100,800,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-SH_v2r2_GW1_s202106302341190_e202107010120160_c202107010202390.nc icec_amsr2_south.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5, GAASP-L1R_v2r2_GW1_s202106292259160_e202106300038160_c202106300112300.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :time_coverage_start = "2021-06-30T23:41:19.004Z" ; + :time_coverage_end = "2021-07-01T01:20:16.940Z" ; + :start_orbit_number = 48513 ; + :end_orbit_number = 48514 ; + :ascend_descend_data_flag = 2 ; + :day_night_data_flag = 2 ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Along_Y_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Bootstrap_Ice_Concentration = + 0, 0, 0, 100, 93, 0, 0, 0, + 0, 94, 100, 91, 95, 84, 0, 0, + 0, _, 100, _, _, _, 97, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 95, 0, + 0, 0, 100, 96, _, _, 0, 0, + 0, 0, 0, 0, 91, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Flags = + 4, 8, 8, 0, 0, 8, 8, 4, + 4, 0, 0, 0, 0, 0, 8, 8, + 8, 120, 0, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 4, 8, 0, 0, 120, 120, 8, 4, + 4, 8, 8, 8, 0, 8, 8, 4, + 4, 4, 4, 4, 4, 4, 4, 4 ; + + Latency = + _, 537, 539, 60, 60, 160, 162, _, + _, 441, 442, 57, 157, 158, 259, 261, + 341, 2000, 54, 2000, 2000, 2000, 257, 358, + 242, 245, 2000, 2000, 2000, 2000, 356, 358, + 242, 146, 2000, 2000, 2000, 2000, 455, 1032, + _, 45, 47, 748, 2000, 2000, 555, _, + _, 44, 45, 751, 652, 653, 557, _, + _, _, _, _, _, _, _, _ ; + + Latitude = + -48.58855, -54.69889, -59.05717, -61.01042, -60.13906, -56.64225, + -51.14237, -44.22784, + -54.69889, -61.90622, -67.46837, -70.16405, -68.94062, -64.3263, -57.65202, + -49.79427, + -59.05717, -67.46837, -74.82165, -79.14207, -77.07041, -70.51806, + -62.41702, -53.64244, + -61.01042, -70.16405, -79.14207, -87.5308, -82.57877, -73.68634, -64.60285, + -55.3238, + -60.13906, -68.94062, -77.07041, -82.57877, -79.79292, -72.22871, + -63.62301, -54.57748, + -56.64225, -64.3263, -70.51806, -73.68634, -72.22871, -66.97952, -59.76062, + -51.52504, + -51.14237, -57.65202, -62.41702, -64.60285, -63.62301, -59.76062, + -53.84447, -46.57473, + -44.22784, -49.79427, -53.64244, -55.3238, -54.57748, -51.52504, -46.57473, + -40.16538 ; + + Longitude = + -45, -34.48949, -20.5069, -3.492593, 14.14168, 29.46407, 41.28101, 49.98034, + -55.51051, -45, -28.56472, -5.07673, 20.14014, 39.43126, 51.95571, 60.02053, + -69.4931, -61.43528, -45, -9.267837, 33.96579, 56.49345, 66.92478, 72.56456, + -86.50741, -84.92327, -80.73216, -45, 76.38319, 83.83407, 86.02327, 87.06625, + -104.1417, -110.1401, -123.9658, -166.3832, 135, 114.0361, 106.0128, + 101.9456, + -119.4641, -129.4313, -146.4935, -173.8341, 155.9639, 135, 122.761, 115.3785, + -131.281, -141.9557, -156.9248, -176.0233, 163.9872, 147.239, 135, 126.3972, + -139.9803, -150.0205, -162.5646, -177.0663, 168.0544, 154.6215, 143.6028, + 135 ; + + NASA_Team_2_Ice_Concentration = + 0, 0, 0, 100, 100, 0, 0, 0, + 0, 100, 100, 100, 100, 99, 0, 0, + 0, _, 100, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 0, 100, 100, _, _, 0, 0, + 0, 0, 0, 0, 100, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 65, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, 0, 7, 0, 0, 0, + 0, 6, 0, 9, 5, 15, 0, 0, + 0, _, 0, _, _, _, 3, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 0, _, _, _, _, 5, 0, + 0, 0, 0, 4, _, _, 0, 0, + 0, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + QM_Num_Grid_Range_25 = 3427 ; + + QM_Num_Grid_Range_50 = 830 ; + + QM_Total_Ice = 148634 ; + + QM_Total_Pixels = 705600 ; + + Range_of_Ice_Concentration = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 0, 0, 1, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 1, _, _, _, _, 0, 0, + 0, 4, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Scan_Time = + _, _, _, _, _, _, + 2018, 4, 15, 16, 22, 42.51182, + 2018, 4, 15, 16, 21, 9.52068, + 2018, 4, 15, 0, 20, 8.284265, + 2018, 4, 15, 0, 19, 36.78727, + 2018, 4, 15, 22, 39, 34.35703, + 2018, 4, 15, 22, 38, 11.86481, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 17, 59, 8.96092, + 2018, 4, 15, 17, 57, 28.47058, + 2018, 4, 15, 0, 22, 18.77182, + 2018, 4, 15, 22, 43, 5.836905, + 2018, 4, 15, 22, 42, 7.342505, + 2018, 4, 15, 21, 1, 4.91956, + 2018, 4, 15, 20, 58, 39.43348, + 2018, 4, 15, 19, 38, 54.88971, + _, _, _, _, _, _, + 2018, 4, 15, 0, 25, 57.75103, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 11, 3, 9.407737, + 2018, 4, 15, 11, 21, 51.98771, + 2018, 4, 15, 14, 17, 33.32495, + 2018, 4, 15, 14, 14, 22.84327, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 24, 8.474729, + 2018, 4, 15, 15, 21, 23.49041, + 2018, 4, 15, 15, 17, 42.32409, + 2018, 4, 15, 15, 53, 38.77679, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 44, 46.54345, + 2018, 4, 15, 9, 8, 10.83098, + _, _, _, _, _, _, + 2018, 4, 15, 0, 34, 26.20252, + 2018, 4, 15, 0, 32, 32.21346, + 2018, 4, 15, 12, 51, 42.2143, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 15, 5, 11.11222, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 9, 36, 8.19276, + 2018, 4, 15, 9, 35, 8.198488, + 2018, 4, 15, 12, 48, 33.2322, + 2018, 4, 15, 14, 27, 37.16796, + 2018, 4, 15, 14, 26, 19.1754, + 2018, 4, 15, 14, 3, 8.123897, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/icec_amsr2_south_2.cdl b/test/soca/testdata/icec_amsr2_south_2.cdl new file mode 100644 index 000000000..af5a60251 --- /dev/null +++ b/test/soca/testdata/icec_amsr2_south_2.cdl @@ -0,0 +1,306 @@ +netcdf icec_amsr2_south_2 { +dimensions: + Number_of_X_Dimension = 8 ; + Number_of_Y_Dimension = 8 ; + QM = 1 ; + Time_Dimension = 6 ; +variables: + short Across_X_Dimension(Number_of_X_Dimension) ; + Across_X_Dimension:long_name = " Across X Dimension" ; + Across_X_Dimension:coordinates = "" ; + Across_X_Dimension:units = "1" ; + Across_X_Dimension:_FillValue = -9999s ; + short Along_Y_Dimension(Number_of_Y_Dimension) ; + Along_Y_Dimension:long_name = " Along Y Dimension" ; + Along_Y_Dimension:coordinates = "" ; + Along_Y_Dimension:units = "1" ; + Along_Y_Dimension:_FillValue = -9999s ; + int Bootstrap_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Bootstrap_Ice_Concentration:long_name = "Bootstrap Ice Concentration (Fraction)" ; + Bootstrap_Ice_Concentration:units = "1" ; + Bootstrap_Ice_Concentration:coordinates = "Latitude Longitude" ; + Bootstrap_Ice_Concentration:_FillValue = -9999 ; + short Flags(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Flags:long_name = "Quality Flags" ; + Flags:units = "1" ; + Flags:coordinate = "Latitude Longitude" ; + Flags:_FillValue = -9999s ; + Flags:comment = "0: good, 4: SST missing, 8: Weather, 16: spillover, 32: filled, 64: NT2 Missing" ; + int Latency(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latency:long_name = "Time in seconds from most current time" ; + Latency:units = "seconds" ; + Latency:coordinate = "Latitude Longitude" ; + Latency:_FillValue = -9999 ; + float Latitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Latitude:long_name = "Latitude for EASE-2 grid" ; + Latitude:units = "degrees_north" ; + Latitude:valid_range = -90.f, 90.f ; + Latitude:_FillValue = -9999.f ; + Latitude:standard_name = "latitude" ; + float Longitude(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Longitude:long_name = "Longitude for EASE-2 grid" ; + Longitude:units = "degrees_east" ; + Longitude:valid_range = -180.f, 180.f ; + Longitude:_FillValue = -9999.f ; + Longitude:standard_name = "longitude" ; + int NASA_Team_2_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Ice_Concentration:long_name = "NASA Team 2 Ice Concentration (Fraction)" ; + NASA_Team_2_Ice_Concentration:units = "1" ; + NASA_Team_2_Ice_Concentration:coordinates = "Latitude Longitude" ; + NASA_Team_2_Ice_Concentration:_FillValue = -9999 ; + int NASA_Team_2_Multiyear_Ice(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NASA_Team_2_Multiyear_Ice:long_name = "NASA Team 2 Multiyear Ice (Fraction)" ; + NASA_Team_2_Multiyear_Ice:units = "1" ; + NASA_Team_2_Multiyear_Ice:coordinates = "Latitude Longitude" ; + NASA_Team_2_Multiyear_Ice:_FillValue = -9999 ; + int NT2_minus_Bootstrap(Number_of_Y_Dimension, Number_of_X_Dimension) ; + NT2_minus_Bootstrap:long_name = "NT2 minus Bootstrap (Fraction)" ; + NT2_minus_Bootstrap:units = "1" ; + NT2_minus_Bootstrap:coordinates = "Latitude Longitude" ; + NT2_minus_Bootstrap:_FillValue = -9999 ; + float QM_Num_Grid_Range_25(QM) ; + QM_Num_Grid_Range_25:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 25%" ; + QM_Num_Grid_Range_25:units = "1" ; + QM_Num_Grid_Range_25:_FillValue = -9999.f ; + float QM_Num_Grid_Range_50(QM) ; + QM_Num_Grid_Range_50:long_name = "Quality Monitoring: Number of Grid Boxes with Range > 50%" ; + QM_Num_Grid_Range_50:units = "1" ; + QM_Num_Grid_Range_50:_FillValue = -9999.f ; + float QM_Total_Ice(QM) ; + QM_Total_Ice:long_name = "Quality Monitoring: Total Ice Extent" ; + QM_Total_Ice:units = "1" ; + QM_Total_Ice:_FillValue = -9999.f ; + float QM_Total_Pixels(QM) ; + QM_Total_Pixels:long_name = "Quality Monitoring: Total Number of Pixels" ; + QM_Total_Pixels:units = "1" ; + QM_Total_Pixels:_FillValue = -9999.f ; + int Range_of_Ice_Concentration(Number_of_Y_Dimension, Number_of_X_Dimension) ; + Range_of_Ice_Concentration:long_name = "Range of Ice Concentration (Fraction)" ; + Range_of_Ice_Concentration:units = "1" ; + Range_of_Ice_Concentration:coordinates = "Latitude Longitude" ; + Range_of_Ice_Concentration:_FillValue = -9999 ; + float Scan_Time(Number_of_Y_Dimension, Number_of_X_Dimension, Time_Dimension) ; + Scan_Time:long_name = "Scan line Start Time 6-D for (YYYY, MM, DD, HH, MM, SS) in GMT" ; + Scan_Time:units = "1" ; + Scan_Time:_FillValue = -9999.f ; + +// global attributes: + :Conventions = "CF-1.5" ; + :Metadata_Conventions = "CF-1.5, Unidata Datasset Discovery v1.0" ; + :standard_name_vocabulary = "CF Standard Name Table (version 17, 24 March 2011)" ; + :project = "NPP Data Exploitation: NOAA GCOM-W1 AMSR2" ; + :title = "AMSR2_SEAICE_SH" ; + :summary = "GCOM Seaice Southern Hemisphere Products" ; + :institution = "DOC/NOAA/NESDIS/OSPO > Office of Satellite and Product Operations, NESDIS, NOAA, U.S. Department of Commerce" ; + :naming_authority = "gov.noaa.nesdis.nde" ; + :platform_name = "GCOM-W1" ; + :instrument_name = "AMSR2" ; + :creator_name = "DOC/NOAA/NESDIS/STAR > IOSSPDT Algorithm Team, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :creator_email = "espcoperations@noaa.gov" ; + :creator_url = "http://www.star.nesdis.noaa.gov" ; + :publisher_name = "DOC/NOAA/NESDIS/NDE > NPP Data Exploitation, Center for Satellite Applications and Research, NESDIS, NOAA, U.S. Department of Commerce" ; + :publisher_email = "espcoperations@noaa.gov" ; + :publisher_url = "http://www.ospo.noaa.gov/" ; + :references = "Contact the OSPO PAL to request the ATBD." ; + :processing_level = "NOAA Level 2 data" ; + :cdm_data_type = "Grid" ; + :geospatial_lat_min = -89.9367f ; + :geospatial_lat_max = -34.6223f ; + :geospatial_lon_min = -179.932f ; + :geospatial_lon_max = 179.932f ; + :geospatial_lat_units = "degrees_north" ; + :geospatial_lon_units = "degrees_east" ; + :date_created = "2021-07-01T03:40:15Z" ; + :id = "d4079f1c-3c57-46b3-ac1d-d650937aebde" ; + :Metadata_Link = "AMSR2-SEAICE-SH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc" ; + :history = "Mon Sep 25 15:42:01 2023: ncks -d Number_of_X_Dimension,100,800,100 -d Number_of_Y_Dimension,100,800,100 /scratch1/NCEPDEV/stmp4/Shastri.Paturi/forAndrew/gdas.20210701/00/icec/AMSR2-SEAICE-SH_v2r2_GW1_s202107010120180_e202107010259170_c202107010340150.nc icec_amsr2_south_2.nc\nCreated by GAASP Version 2.0, Release 2.0" ; + :source = "GAASP-L1R_v2r2_GW1_s202107010120180_e202107010259170_c202107010336520.h5, GAASP-L1R_v2r2_GW1_s202106302341190_e202107010120160_c202107010159060.h5, GAASP-L1R_v2r2_GW1_s202106302202190_e202106302341170_c202107010019490.h5, GAASP-L1R_v2r2_GW1_s202106302020180_e202106302202180_c202106302237500.h5, GAASP-L1R_v2r2_GW1_s202106301838190_e202106302020170_c202106302057270.h5, GAASP-L1R_v2r2_GW1_s202106301659190_e202106301838170_c202106301916000.h5, GAASP-L1R_v2r2_GW1_s202106301520170_e202106301659180_c202106301736190.h5, GAASP-L1R_v2r2_GW1_s202106301341160_e202106301520150_c202106301558020.h5, GAASP-L1R_v2r2_GW1_s202106301205160_e202106301341140_c202106301416190.h5, GAASP-L1R_v2r2_GW1_s202106301026170_e202106301205150_c202106301238160.h5, GAASP-L1R_v2r2_GW1_s202106300847160_e202106301026160_c202106301101410.h5, GAASP-L1R_v2r2_GW1_s202106300711170_e202106300847150_c202106300921380.h5, GAASP-L1R_v2r2_GW1_s202106300532170_e202106300711150_c202106300744420.h5, GAASP-L1R_v2r2_GW1_s202106300356160_e202106300532160_c202106300607150.h5, GAASP-L1R_v2r2_GW1_s202106300217170_e202106300356150_c202106300428400.h5, GAASP-L1R_v2r2_GW1_s202106300038170_e202106300217150_c202106300252420.h5" ; + :production_site = "NSOF" ; + :production_environment = "OE" ; + :time_coverage_start = "2021-07-01T01:20:18.440Z" ; + :time_coverage_end = "2021-07-01T02:59:17.876Z" ; + :start_orbit_number = 48514 ; + :end_orbit_number = 48515 ; + :ascend_descend_data_flag = 2 ; + :day_night_data_flag = 2 ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + Across_X_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Along_Y_Dimension = 101, 201, 301, 401, 501, 601, 701, 801 ; + + Bootstrap_Ice_Concentration = + 0, 0, 0, 100, 94, 0, 0, 0, + 0, 94, 100, 82, 95, 84, 0, 0, + 0, _, 99, _, _, _, 97, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 99, _, _, _, _, 95, 0, + 0, 0, 94, 96, _, _, 0, 0, + 0, 0, 0, 0, 91, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Flags = + 4, 8, 8, 0, 0, 8, 8, 4, + 8, 0, 0, 0, 0, 0, 8, 8, + 8, 120, 0, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 8, 0, 120, 120, 120, 120, 0, 8, + 4, 8, 0, 0, 120, 120, 8, 8, + 4, 8, 8, 8, 0, 8, 8, 4, + 4, 4, 4, 4, 4, 4, 4, 4 ; + + Latency = + _, 636, 61, 159, 159, 259, 261, _, + 537, 540, 58, 156, 256, 257, 358, 360, + 440, 2000, 56, 2000, 2000, 2000, 356, 457, + 341, 344, 2000, 2000, 2000, 2000, 455, 457, + 341, 245, 2000, 2000, 2000, 2000, 554, 1131, + _, 144, 48, 48, 2000, 2000, 654, 557, + _, 143, 45, 46, 751, 752, 656, _, + _, _, _, _, _, _, _, _ ; + + Latitude = + -48.58855, -54.69889, -59.05717, -61.01042, -60.13906, -56.64225, + -51.14237, -44.22784, + -54.69889, -61.90622, -67.46837, -70.16405, -68.94062, -64.3263, -57.65202, + -49.79427, + -59.05717, -67.46837, -74.82165, -79.14207, -77.07041, -70.51806, + -62.41702, -53.64244, + -61.01042, -70.16405, -79.14207, -87.5308, -82.57877, -73.68634, -64.60285, + -55.3238, + -60.13906, -68.94062, -77.07041, -82.57877, -79.79292, -72.22871, + -63.62301, -54.57748, + -56.64225, -64.3263, -70.51806, -73.68634, -72.22871, -66.97952, -59.76062, + -51.52504, + -51.14237, -57.65202, -62.41702, -64.60285, -63.62301, -59.76062, + -53.84447, -46.57473, + -44.22784, -49.79427, -53.64244, -55.3238, -54.57748, -51.52504, -46.57473, + -40.16538 ; + + Longitude = + -45, -34.48949, -20.5069, -3.492593, 14.14168, 29.46407, 41.28101, 49.98034, + -55.51051, -45, -28.56472, -5.07673, 20.14014, 39.43126, 51.95571, 60.02053, + -69.4931, -61.43528, -45, -9.267837, 33.96579, 56.49345, 66.92478, 72.56456, + -86.50741, -84.92327, -80.73216, -45, 76.38319, 83.83407, 86.02327, 87.06625, + -104.1417, -110.1401, -123.9658, -166.3832, 135, 114.0361, 106.0128, + 101.9456, + -119.4641, -129.4313, -146.4935, -173.8341, 155.9639, 135, 122.761, 115.3785, + -131.281, -141.9557, -156.9248, -176.0233, 163.9872, 147.239, 135, 126.3972, + -139.9803, -150.0205, -162.5646, -177.0663, 168.0544, 154.6215, 143.6028, + 135 ; + + NASA_Team_2_Ice_Concentration = + 0, 0, 0, 100, 100, 0, 0, 0, + 0, 100, 100, 100, 100, 99, 0, 0, + 0, _, 100, _, _, _, 100, 0, + 0, 99, _, _, _, _, 100, 0, + 0, 100, _, _, _, _, 100, 0, + 0, 0, 100, 100, _, _, 0, 0, + 0, 0, 0, 0, 100, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NASA_Team_2_Multiyear_Ice = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 57, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + NT2_minus_Bootstrap = + 0, 0, 0, 0, 6, 0, 0, 0, + 0, 6, 0, 18, 5, 15, 0, 0, + 0, _, 1, _, _, _, 3, 0, + 0, 0, _, _, _, _, 0, 0, + 0, 1, _, _, _, _, 5, 0, + 0, 0, 6, 4, _, _, 0, 0, + 0, 0, 0, 0, 9, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + QM_Num_Grid_Range_25 = 3269 ; + + QM_Num_Grid_Range_50 = 775 ; + + QM_Total_Ice = 148706 ; + + QM_Total_Pixels = 705600 ; + + Range_of_Ice_Concentration = + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 0, 0, 1, 0, 0, + 0, _, 0, _, _, _, 0, 0, + 0, 1, _, _, _, _, 0, 0, + 0, 4, _, _, _, _, 0, 0, + 0, 0, 0, 0, _, _, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0 ; + + Scan_Time = + _, _, _, _, _, _, + 2018, 4, 15, 10, 22, 42.51182, + 2018, 4, 15, 11, 57, 58.72683, + 2018, 4, 15, 10, 20, 8.284265, + 2018, 4, 15, 10, 19, 36.78727, + 2018, 4, 15, 12, 39, 34.35703, + 2018, 4, 15, 12, 38, 11.86481, + _, _, _, _, _, _, + 2018, 4, 15, 11, 1, 22.44809, + 2018, 4, 15, 10, 59, 8.96092, + 2018, 4, 15, 12, 0, 28.71251, + 2018, 4, 15, 10, 22, 18.77182, + 2018, 4, 15, 12, 43, 5.836905, + 2018, 4, 15, 12, 42, 7.342505, + 2018, 4, 15, 11, 1, 4.91956, + 2018, 4, 15, 10, 58, 39.43348, + 2018, 4, 15, 13, 38, 54.88971, + _, _, _, _, _, _, + 2018, 4, 15, 2, 2, 57.19851, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 21, 3, 9.407737, + 2018, 4, 15, 19, 21, 51.98771, + 2018, 4, 15, 21, 17, 33.32495, + 2018, 4, 15, 21, 14, 22.84327, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 19, 24, 8.474729, + 2018, 4, 15, 19, 21, 23.49041, + 2018, 4, 15, 21, 17, 42.32409, + 2018, 4, 15, 22, 53, 38.77679, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 12, 44, 46.54345, + 2018, 4, 15, 8, 8, 10.83098, + _, _, _, _, _, _, + 2018, 4, 15, 0, 34, 26.20252, + 2018, 4, 15, 2, 10, 45.15401, + 2018, 4, 15, 2, 10, 49.65358, + _, _, _, _, _, _, + _, _, _, _, _, _, + 2018, 4, 15, 14, 5, 11.11222, + 2018, 4, 15, 14, 41, 39.06126, + _, _, _, _, _, _, + 2018, 4, 15, 0, 36, 8.19276, + 2018, 4, 15, 2, 14, 3.135001, + 2018, 4, 15, 2, 13, 6.140425, + 2018, 4, 15, 14, 27, 37.16796, + 2018, 4, 15, 14, 26, 19.1754, + 2018, 4, 15, 14, 3, 8.123897, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _, + _, _, _, _, _, _ ; +} diff --git a/test/soca/testdata/rads_adt_3a_2018105.cdl b/test/soca/testdata/rads_adt_3a_2018105.cdl new file mode 100644 index 000000000..3e08dd0f1 --- /dev/null +++ b/test/soca/testdata/rads_adt_3a_2018105.cdl @@ -0,0 +1,110 @@ +netcdf rads_adt_3a_2021181 { +dimensions: + time = UNLIMITED ; // (11 currently) +variables: + int adt_egm2008(time) ; + adt_egm2008:_FillValue = 2147483647 ; + adt_egm2008:long_name = "absolute dynamic topography (EGM2008)" ; + adt_egm2008:standard_name = "absolute_dynamic_topography_egm2008" ; + adt_egm2008:units = "m" ; + adt_egm2008:scale_factor = 0.0001 ; + adt_egm2008:coordinates = "lon lat" ; + int adt_xgm2016(time) ; + adt_xgm2016:_FillValue = 2147483647 ; + adt_xgm2016:long_name = "absolute dynamic topography (XGM2016)" ; + adt_xgm2016:standard_name = "absolute_dynamic_topography_xgm2016" ; + adt_xgm2016:units = "m" ; + adt_xgm2016:scale_factor = 0.0001 ; + adt_xgm2016:coordinates = "lon lat" ; + int cycle(time) ; + cycle:_FillValue = 2147483647 ; + cycle:long_name = "cycle number" ; + cycle:field = 9905s ; + int lat(time) ; + lat:_FillValue = 2147483647 ; + lat:long_name = "latitude" ; + lat:standard_name = "latitude" ; + lat:units = "degrees_north" ; + lat:scale_factor = 1.e-06 ; + lat:field = 201s ; + lat:comment = "Positive latitude is North latitude, negative latitude is South latitude" ; + int lon(time) ; + lon:_FillValue = 2147483647 ; + lon:long_name = "longitude" ; + lon:standard_name = "longitude" ; + lon:units = "degrees_east" ; + lon:scale_factor = 1.e-06 ; + lon:field = 301s ; + lon:comment = "East longitude relative to Greenwich meridian" ; + int pass(time) ; + pass:_FillValue = 2147483647 ; + pass:long_name = "pass number" ; + pass:field = 9906s ; + short sla(time) ; + sla:_FillValue = 32767s ; + sla:long_name = "sea level anomaly" ; + sla:standard_name = "sea_surface_height_above_sea_level" ; + sla:units = "m" ; + sla:quality_flag = "swh sig0 range_rms range_numval flags swh_rms sig0_rms" ; + sla:scale_factor = 0.0001 ; + sla:coordinates = "lon lat" ; + sla:field = 0s ; + sla:comment = "Sea level determined from satellite altitude - range - all altimetric corrections" ; + double time_dtg(time) ; + time_dtg:long_name = "time_dtg" ; + time_dtg:standard_name = "time_dtg" ; + time_dtg:units = "yyyymmddhhmmss" ; + time_dtg:coordinates = "lon lat" ; + time_dtg:comment = "UTC time formatted as yyyymmddhhmmss" ; + double time_mjd(time) ; + time_mjd:long_name = "Modified Julian Days" ; + time_mjd:standard_name = "time" ; + time_mjd:units = "days since 1858-11-17 00:00:00 UTC" ; + time_mjd:field = 105s ; + time_mjd:comment = "UTC time of measurement expressed in Modified Julian Days" ; + +// global attributes: + :Conventions = "CF-1.7" ; + :title = "RADS 4 pass file" ; + :institution = "EUMETSAT / NOAA / TU Delft" ; + :source = "radar altimeter" ; + :references = "RADS Data Manual, Version 4.2 or later" ; + :featureType = "trajectory" ; + :ellipsoid = "TOPEX" ; + :ellipsoid_axis = 6378136.3 ; + :ellipsoid_flattening = 0.00335281317789691 ; + :filename = "rads_adt_3a_2021181.nc" ; + :mission_name = "SNTNL-3A" ; + :mission_phase = "a" ; + :log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc: RAW data from" ; + :history = "Mon Sep 25 17:01:30 2023: ncks -d time,0,10 rads_adt_3a_2021181.nc rads_adt_3a_2021181.ncn\n", + "2021-07-01 21:14:30 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -C1,1000 -S3a -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3a_2021181.nc" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + adt_egm2008 = 1674, 2176, 1671, 2320, 2011, 2349, 2139, 2174, 2462, 2441, + 2487 ; + + adt_xgm2016 = 1621, 2300, 1938, 2652, 2093, 2246, 1948, 1976, 2406, 2469, + 2649 ; + + cycle = 73, 73, 73, 73, 73, 73, 73, 73, 73, 73, 73 ; + + lat = -43973134, -43915104, -43857071, -43799035, -43740996, -43682953, + -43624907, -43566858, -43508805, -43450749, -43392690 ; + + lon = -21550109, -21571346, -21592549, -21613720, -21634859, -21655964, + -21677037, -21698078, -21719086, -21740062, -21761007 ; + + pass = 517, 517, 517, 517, 517, 517, 517, 517, 517, 517, 517 ; + + sla = 75, 523, -338, 671, 104, 513, 128, 108, 424, 322, 497 ; + + time_dtg = 20180415000000, 20180415000001, 20180415000002, 20180415000003, + 20180415000004, 20180415000005, 20180415000006, 20180415000007, + 20180415000008, 20180415000009, 20180415000010 ; + + time_mjd = 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5 ; +} diff --git a/test/soca/testdata/rads_adt_3b_2018105.cdl b/test/soca/testdata/rads_adt_3b_2018105.cdl new file mode 100644 index 000000000..c43aef2f2 --- /dev/null +++ b/test/soca/testdata/rads_adt_3b_2018105.cdl @@ -0,0 +1,110 @@ +netcdf rads_adt_3b_2021181 { +dimensions: + time = UNLIMITED ; // (11 currently) +variables: + int adt_egm2008(time) ; + adt_egm2008:_FillValue = 2147483647 ; + adt_egm2008:long_name = "absolute dynamic topography (EGM2008)" ; + adt_egm2008:standard_name = "absolute_dynamic_topography_egm2008" ; + adt_egm2008:units = "m" ; + adt_egm2008:scale_factor = 0.0001 ; + adt_egm2008:coordinates = "lon lat" ; + int adt_xgm2016(time) ; + adt_xgm2016:_FillValue = 2147483647 ; + adt_xgm2016:long_name = "absolute dynamic topography (XGM2016)" ; + adt_xgm2016:standard_name = "absolute_dynamic_topography_xgm2016" ; + adt_xgm2016:units = "m" ; + adt_xgm2016:scale_factor = 0.0001 ; + adt_xgm2016:coordinates = "lon lat" ; + int cycle(time) ; + cycle:_FillValue = 2147483647 ; + cycle:long_name = "cycle number" ; + cycle:field = 9905s ; + int lat(time) ; + lat:_FillValue = 2147483647 ; + lat:long_name = "latitude" ; + lat:standard_name = "latitude" ; + lat:units = "degrees_north" ; + lat:scale_factor = 1.e-06 ; + lat:field = 201s ; + lat:comment = "Positive latitude is North latitude, negative latitude is South latitude" ; + int lon(time) ; + lon:_FillValue = 2147483647 ; + lon:long_name = "longitude" ; + lon:standard_name = "longitude" ; + lon:units = "degrees_east" ; + lon:scale_factor = 1.e-06 ; + lon:field = 301s ; + lon:comment = "East longitude relative to Greenwich meridian" ; + int pass(time) ; + pass:_FillValue = 2147483647 ; + pass:long_name = "pass number" ; + pass:field = 9906s ; + short sla(time) ; + sla:_FillValue = 32767s ; + sla:long_name = "sea level anomaly" ; + sla:standard_name = "sea_surface_height_above_sea_level" ; + sla:units = "m" ; + sla:quality_flag = "swh sig0 range_rms range_numval flags swh_rms sig0_rms" ; + sla:scale_factor = 0.0001 ; + sla:coordinates = "lon lat" ; + sla:field = 0s ; + sla:comment = "Sea level determined from satellite altitude - range - all altimetric corrections" ; + double time_dtg(time) ; + time_dtg:long_name = "time_dtg" ; + time_dtg:standard_name = "time_dtg" ; + time_dtg:units = "yyyymmddhhmmss" ; + time_dtg:coordinates = "lon lat" ; + time_dtg:comment = "UTC time formatted as yyyymmddhhmmss" ; + double time_mjd(time) ; + time_mjd:long_name = "Modified Julian Days" ; + time_mjd:standard_name = "time" ; + time_mjd:units = "days since 1858-11-17 00:00:00 UTC" ; + time_mjd:field = 105s ; + time_mjd:comment = "UTC time of measurement expressed in Modified Julian Days" ; + +// global attributes: + :Conventions = "CF-1.7" ; + :title = "RADS 4 pass file" ; + :institution = "EUMETSAT / NOAA / TU Delft" ; + :source = "radar altimeter" ; + :references = "RADS Data Manual, Version 4.2 or later" ; + :featureType = "trajectory" ; + :ellipsoid = "TOPEX" ; + :ellipsoid_axis = 6378136.3 ; + :ellipsoid_flattening = 0.00335281317789691 ; + :filename = "rads_adt_3b_2021181.nc" ; + :mission_name = "SNTNL-3B" ; + :mission_phase = "b" ; + :log01 = "2021-07-01 | /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc: RAW data from" ; + :history = "Mon Sep 25 17:01:31 2023: ncks -d time,0,10 rads_adt_3b_2021181.nc rads_adt_3b_2021181.ncn\n", + "2021-07-01 21:31:13 : /Users/rads/bin/rads2nc --ymd=20210630000000,20210701000000 -S3b -Vsla,adt_egm2008,adt_xgm2016,time_mjd,time_dtg,lon,lat,cycle,pass -X/Users/rads/cron/xgm2016 -X/Users/rads/cron/adt -X/Users/rads/cron/time_dtg -o/Users/rads/adt/2021/181/rads_adt_3b_2021181.nc" ; + :NCO = "netCDF Operators version 5.0.6 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + adt_egm2008 = 6505, 7307, 6026, 5871, 5561, 5246, 4981, 4661, 4391, 4409, + 4283 ; + + adt_xgm2016 = 3136, 4247, 4498, 4515, 4579, 4741, 4731, 4351, 4016, 3860, + 3887 ; + + cycle = 54, 54, 54, 54, 54, 54, 54, 54, 54, 54, 54 ; + + lat = 59732735, 59505896, 58767628, 58710774, 58653911, 58597040, 58540159, + 58483270, 58426372, 58369465, 58312550 ; + + lon = 163417447, 163262248, 162770394, 162733328, 162696369, 162659516, + 162622770, 162586128, 162549591, 162513159, 162476830 ; + + pass = 232, 232, 232, 232, 232, 232, 232, 232, 232, 232, 232 ; + + sla = 4578, 5072, 1437, 756, 254, 91, -2, -170, -484, -313, -244 ; + + time_dtg = 20180415000629, 20180415000633, 20180415000646, 20180415000647, + 20180415000648, 20180415000649, 20180415000650, 20180415000651, + 20180415000652, 20180415000653, 20180415000654 ; + + time_mjd = 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5, + 58223.5, 58223.5, 58223.5, 58223.5 ; +} diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 949526019..8610af15c 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -25,6 +25,8 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index 469fd4a3c..ef7f73699 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -22,6 +22,13 @@ elif [[ $MACHINE_ID = orion* ]] ; then fi module purge +elif [[ $MACHINE_ID = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/lmod/init/bash + fi + module purge + elif [[ $MACHINE_ID = s4* ]] ; then # We are on SSEC Wisconsin S4 if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/ush/soca/run_jjobs.py b/ush/soca/run_jjobs.py index 4e0212da2..f9b73c845 100755 --- a/ush/soca/run_jjobs.py +++ b/ush/soca/run_jjobs.py @@ -6,7 +6,7 @@ import argparse from datetime import datetime, timedelta -machines = {"container", "hera", "orion"} +machines = {"container", "hera", "orion", "hercules"} # Assume the default conda environement is gdassapp ENVS = {'JGDAS_GLOBAL_OCEAN_ANALYSIS_VRFY': 'eva'} diff --git a/ush/ufsda/soca_utils.py b/ush/ufsda/soca_utils.py deleted file mode 100644 index 83bfb9d7b..000000000 --- a/ush/ufsda/soca_utils.py +++ /dev/null @@ -1,118 +0,0 @@ -import xarray -import glob -import numpy as np -import sys -import os -import logging -import shutil - -# TODO: We might want to revisit this in the future -# Try to resolve the location of pyioda, assuming there are only 2 places where this -# script can exist (build/ush/ufsda or /ush/ufsda) -from pathlib import Path -jedilib = Path(os.path.join(Path(__file__).parent.absolute(), '../..', 'lib')) -if not jedilib.is_dir(): - jedilib = Path(os.path.join(Path(__file__).parent.absolute(), '../../build', 'lib')) -pyver = 'python3.'+str(sys.version_info[1]) -pyioda_lib = Path(os.path.join(jedilib, pyver, 'pyioda')).resolve() -pyiodaconv_lib = Path(os.path.join(jedilib, 'pyiodaconv')).resolve() -sys.path.append(str(pyioda_lib)) -sys.path.append(str(pyiodaconv_lib)) -import pyiodaconv.ioda_conv_engines as iconv -from pyiodaconv.orddicts import DefaultOrderedDict - -__all__ = ['concatenate_ioda'] - -logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', - level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S') - -marine_obs_names = ['seaSurfaceTemperature', - 'absoluteDynamicTopography', - 'seaIceFraction', - 'seaIceFreeboard', - 'waterTemperature', - 'salinity', - 'seaSurfaceSalinity'] - -iodatypes = {"ObsError": 'float32', - "ObsValue": 'float32', - "PreQC": 'int32'} - - -def obs_specs(iodafname): - with xarray.open_dataset(iodafname, group='MetaData', decode_cf=False) as ds: - timeref = ds['dateTime'].attrs['units'] - - with xarray.open_dataset(iodafname, group='ObsValue') as ds: - for varname in ds.variables: - if varname in marine_obs_names: - logging.info(f"Found {varname} in {iodafname}.") - var = ds[varname] - return varname, var.attrs['units'], timeref - - raise Exception(f"No known obs type in {iodafname}.") - - -def concatenate_ioda(iodafname, wildcard=".*", output_suffix="", clean=False): - flist = glob.glob(f"{iodafname}{wildcard}") - flist.sort() - nfiles = len(flist) - if nfiles == 0: - logging.info(f"No files to concatenate.") - return - - if len(flist) == 1: - logging.info(f"Only file is {flist[0]}, rename to {iodafname}. No need to concatenate.") - shutil.move(flist[0], iodafname+output_suffix) - return - - logging.info(f"Concatenating {nfiles} files from globbing {iodafname}.*") - - # Identify variable and unit - obsvarname, units, timeref = obs_specs(flist[0]) - - # Concatenate stuff outside of groups (Location dimensions and variables) - ds = xarray.concat([xarray.open_dataset(f) for f in flist], dim='Location') - - # Concatenate all groups except MetaData - outdata = {} - for group in ["ObsError", "ObsValue", "PreQC"]: - ds = xarray.concat([xarray.open_dataset(f, group=group) for f in flist], dim='Location') - outdata[(obsvarname, group)] = ds[obsvarname].astype(iodatypes[group]) - - # Concatenate metadata - ds = xarray.concat([xarray.open_dataset(f, group="MetaData", decode_cf=False) for f in flist], dim='Location') - for k in list(ds.keys()): - outdata[(k, "MetaData")] = ds[k] - - # Setup the IODA writer - varattrs = DefaultOrderedDict(lambda: DefaultOrderedDict(dict)) - - # obs attributes - varattrs[(obsvarname, "ObsValue")]['units'] = units - varattrs[(obsvarname, "ObsError")]['units'] = units - - # meta data attributes - varattrs[('latitude', 'MetaData')]['units'] = 'degrees_north' - varattrs[('longitude', 'MetaData')]['units'] = 'degrees_east' - varattrs[('dateTime', 'MetaData')]['units'] = timeref - locationkeylist = [("latitude", "float", "degrees_north"), - ("longitude", "float", "degrees_east"), - ("dateTime", "long", timeref)] - - vardims = {'': ['Location']} - nlocs = ds.dims['Location'] - dimdict = {} - dimdict['Location'] = nlocs - globalattrs = {} - - # Write - writer = iconv.IodaWriter(iodafname+output_suffix, locationkeylist, dimdict) - writer.BuildIoda(outdata, vardims, varattrs, globalattrs) - - # Remove files - if clean: - for file in flist: - os.remove(file) - - return diff --git a/ush/ufsda/stage.py b/ush/ufsda/stage.py index 84c83a2ef..de0962a0e 100644 --- a/ush/ufsda/stage.py +++ b/ush/ufsda/stage.py @@ -12,7 +12,6 @@ import glob import numpy as np from wxflow import YAMLFile, parse_yaml, parse_j2yaml -import ufsda.soca_utils __all__ = ['atm_background', 'atm_obs', 'bias_obs', 'background', 'background_ens', 'fv3jedi', 'obs', 'berror', 'gdas_fix', 'gdas_single_cycle']