Merge branch 'develop' into feature/stable-nightly

build.sh

@@ -70,7 +70,7 @@ while getopts "p:t:c:hvdfa" opt; do
 done
 
 case ${BUILD_TARGET} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Building GDASApp on $BUILD_TARGET"
     source $dir_root/ush/module-setup.sh
     module use $dir_root/modulefiles

modulefiles/GDAS/hercules.lua

@@ -0,0 +1,98 @@
+help([[
+Load environment for running the GDAS application with Intel compilers and MPI.
+]])
+
+local pkgName    = myModuleName()
+local pkgVersion = myModuleVersion()
+local pkgNameVer = myModuleFullName()
+
+prepend_path("MODULEPATH", '/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core')
+prepend_path("MODULEPATH", '/work2/noaa/da/python/opt/modulefiles/stack')
+
+-- below two lines get us access to the spack-stack modules
+load("stack-intel/2021.9.0")
+load("stack-intel-oneapi-mpi/2021.9.0")
+--load("stack-python/3.10.8")
+-- JCSDA has 'jedi-fv3-env/unified-dev', but we should load these manually as needed
+load("cmake/3.23.1")
+load("curl/8.1.2")
+load("zlib/1.2.13")
+load("git/2.40.0")
+--load("pkg-config/0.27.1")
+load("hdf5/1.14.0")
+load("parallel-netcdf/1.12.2")
+load("netcdf-c/4.9.2")
+load("nccmp/1.9.0.1")
+load("netcdf-fortran/4.6.0")
+load("nco/5.0.6")
+load("parallelio/2.5.10")
+load("wget/1.21.1")
+load("boost/1.78")
+load("bufr/12.0.1")
+load("git-lfs/3.1.2")
+load("ecbuild/3.7.2")
+load("openjpeg/2.4.0")
+load("eccodes/2.27.0")
+load("eigen/3.4.0")
+load("openblas/0.3.19")
+load("eckit/1.24.4")
+load("fftw/3.3.10")
+load("fckit/0.11.0")
+load("fiat/1.2.0")
+load("ectrans/1.2.0")
+load("atlas/0.35.0")
+load("sp/2.3.3")
+load("gsl-lite/0.37.0")
+load("libjpeg/2.1.0")
+load("krb5/1.20.1")
+load("libtirpc/1.2.6")
+load("hdf/4.2.15")
+load("jedi-cmake/1.4.0")
+load("libpng/1.6.37")
+load("libxt/1.1.5")
+load("libxmu/1.1.4")
+load("libxpm/3.5.12")
+load("libxaw/1.0.13")
+load("udunits/2.2.28")
+load("ncview/2.1.8")
+load("netcdf-cxx4/4.3.1")
+load("py-pybind11/2.8.1")
+--load("crtm/v2.4_jedi")
+load("contrib/0.1")
+load("noaatools/3.1")
+load("rocoto/1.3.5")
+
+load("hpc/1.2.0")
+unload("python/3.10.8")
+unload("python/3.9.2")
+load("miniconda3/4.6.14")
+load("gdasapp/1.0.0")
+
+load("boost/1.78.0")
+load("bufr/12.0.1")
+load("fckit/0.11.0")
+load("atlas/0.35.0")
+load("py-pybind11/2.8.1")
+
+-- below is a hack because of cmake finding the wrong python...
+setenv("CONDA_PREFIX", "/work2/noaa/da/python/opt/core/miniconda3/4.6.14/envs/gdasapp/")
+
+setenv("CC","mpiicc")
+setenv("FC","mpiifort")
+setenv("CXX","mpiicpc")
+local mpiexec = '/opt/slurm/bin/srun'
+local mpinproc = '-n'
+setenv('MPIEXEC_EXEC', mpiexec)
+setenv('MPIEXEC_NPROC', mpinproc)
+
+setenv('R2D2_CONFIG', '/work2/noaa/da/cmartin/GDASApp/R2D2_SHARED/config_orion.yaml')
+setenv("CRTM_FIX","/work2/noaa/da/cmartin/GDASApp/fix/crtm/2.4.0")
+setenv("GDASAPP_TESTDATA","/work2/noaa/da/cmartin/CI/GDASApp/data")
+prepend_path("PATH","/apps/contrib/NCEP/libs/hpc-stack/intel-2018.4/prod_util/1.2.2/bin")
+
+execute{cmd="ulimit -s unlimited",modeA={"load"}}
+
+whatis("Name: ".. pkgName)
+whatis("Version: ".. pkgVersion)
+whatis("Category: GDASApp")
+whatis("Description: Load all libraries needed for GDASApp")

parm/land/letkfoi/letkfoi.yaml

@@ -22,6 +22,7 @@ geometry:
 time window:
   begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
   length: $(LAND_WINDOW_LENGTH)
+  bound to include: begin
 
 background:
    datetime: '{{ current_cycle | to_isotime }}'

parm/soca/obs/config/adt_all.yaml → parm/soca/obs/config/adt_rads_all.yaml

@@ -1,13 +1,13 @@
 obs space:
-  name: adt_all
+  name: adt_rads_all
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_all.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_rads_all.${PDY}${cyc}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/adt_all.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/diags/adt_rads_all.${PDY}${cyc}.nc4
   simulated variables: [absoluteDynamicTopography]
   io pool:
     max pool size: 1

parm/soca/obs/config/sst_metopb_l3u_so025.yaml → parm/soca/obs/config/sst_metopb_l3u.yaml

@@ -1,13 +1,13 @@
 obs space:
-  name: sst_metopb_l3u_so025
+  name: sst_metopb_l3u
   obsdatain:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u_so025.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}sst_metopb_l3u.${PDY}${cyc}.nc4
   obsdataout:
     engine:
       type: H5File
-      obsfile: !ENV ${DATA}/diags/sst_metopb_l3u_so025.${PDY}${cyc}.nc4
+      obsfile: !ENV ${DATA}/diags/sst_metopb_l3u.${PDY}${cyc}.nc4
   simulated variables: [seaSurfaceTemperature]
   io pool:
     max pool size: 1

parm/soca/obs/obs_list.yaml

@@ -1,22 +1,5 @@
 observers:
-- !INC ${OBS_YAML_DIR}/adt_all.yaml
-- !INC ${OBS_YAML_DIR}/adt_j3_egm2008.yaml
-- !INC ${OBS_YAML_DIR}/adt_3a_egm2008.yaml
-- !INC ${OBS_YAML_DIR}/adt_3b_egm2008.yaml
-- !INC ${OBS_YAML_DIR}/adt_c2_egm2008.yaml
-- !INC ${OBS_YAML_DIR}/adt_sa_egm2008.yaml
-- !INC ${OBS_YAML_DIR}/sst_viirs_npp_l3u_so025.yaml
-#- !INC ${OBS_YAML_DIR}/sst_metopa_l3u_so025.yaml
-#- !INC ${OBS_YAML_DIR}/sst_metopb_l3u_so025.yaml
-#- !INC ${OBS_YAML_DIR}/sst_metopc_l3u_so025.yaml
-- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_north.yaml
-#- !INC ${OBS_YAML_DIR}/icec_ssmis_f17_south.yaml
-#- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_north.yaml
-- !INC ${OBS_YAML_DIR}/icec_ssmis_f18_south.yaml
-#- !INC ${OBS_YAML_DIR}/temp_profile_argo.yaml
-#- !INC ${OBS_YAML_DIR}/temp_profile_bathy.yaml
-#- !INC ${OBS_YAML_DIR}/temp_profile_glider.yaml
-#- !INC ${OBS_YAML_DIR}/temp_profile_tesac.yaml
-#- !INC ${OBS_YAML_DIR}/salt_profile_argo.yaml
-#- !INC ${OBS_YAML_DIR}/salt_profile_glider.yaml
-#- !INC ${OBS_YAML_DIR}/salt_profile_tesac.yaml
+- !INC ${OBS_YAML_DIR}/adt_rads_all.yaml
+- !INC ${OBS_YAML_DIR}/sst_metopb_l3u.yaml
+- !INC ${OBS_YAML_DIR}/icec_amsr2_north.yaml
+- !INC ${OBS_YAML_DIR}/icec_amsr2_south.yaml

parm/soca/obsproc/obsproc_config.yaml

@@ -12,14 +12,30 @@ observations:
     output file: sss_smos.ioda.nc
     obsproc regex: SM_OPER_MIR_OSUDP2_????????T??????_????????T??????_700_001_1.nc
 - obs space:
-    name: adt_3b_egm2008
+    name: adt_rads_all
     obsproc subdir: ADT
-    obsproc regex: rads_adt_3b_???????.nc
+    obsproc regex: rads_adt_??_???????.nc
     provider: RADS
-    output file: adt_all.nc4
+    output file: adt_rads_all.nc4
 - obs space:
     name: icec_amsr2_north
     provider: AMSR2
     obsproc subdir: icec
-    output file: icec_amsr2_north_1.ioda.nc
+    output file: icec_amsr2_north.ioda.nc
     obsproc regex: AMSR2-SEAICE-NH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: icec_amsr2_south
+    provider: AMSR2
+    obsproc subdir: icec
+    output file: icec_amsr2_south.ioda.nc
+    obsproc regex: AMSR2-SEAICE-SH_v2r2_GW1_s???????????????_e???????????????_c???????????????.nc
+- obs space:
+    name: sst_metopb_l3u
+    provider: GHRSST
+    obsproc subdir: 'sst'
+    output file: sst_metopb_l3u.ioda.nc
+    obsproc regex: '??????????????-STAR-L3U_GHRSST-SSTsubskin-AVHRRF_MB-ACSPO_V2.80-v02.0-fv01.0.nc'
+    bounds:
+      units: C
+      min: -3.0
+      max: 50.0

scripts/exgdas_global_marine_analysis_prep.py

@@ -225,7 +225,7 @@ def find_clim_ens(input_date):
 
 logging.info(f"---------------- Setup runtime environement")
 
-comin_obs = os.getenv('COMIN_OBS')  # R2D2 DB for now
+comin_obs = os.getenv('COMIN_OBS')
 anl_dir = os.getenv('DATA')
 staticsoca_dir = os.getenv('SOCA_INPUT_FIX_DIR')
 if os.getenv('DOHYBVAR') == "YES":
@@ -268,11 +268,6 @@ def find_clim_ens(input_date):
 stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig.get)
 stage_cfg = Template.substitute_structure(stage_cfg, TemplateConstants.DOLLAR_PARENTHESES, envconfig.get)
 
-# concatenate altimeters into one obs space
-# TODO (SAMG)temporary, move this into the obs procecing eventually
-adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt"
-ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True)
-
 # get the list of observations
 obs_files = []
 for ob in stage_cfg['observations']['observers']:

test/atm/global-workflow/jjob_ens_final.sh

@@ -33,22 +33,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_FINALIZE
 fi

test/atm/global-workflow/jjob_ens_init.sh

@@ -38,11 +38,11 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
@@ -69,15 +69,15 @@ dpath=gdas.$PDY/$cyc/obs
 mkdir -p $COM_OBS
 flist="amsua_n19.$CDATE.nc4 sondes.$CDATE.nc4"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${oprefix}.$file $COM_OBS/${oprefix}.$file
 done
 
 # Link radiance bias correction files
 dpath=gdas.$gPDY/$gcyc/analysis/atmos
 mkdir -p $COM_ATMOS_ANALYSIS_PREV
 flist="amsua_n19.satbias.nc4 amsua_n19.satbias_cov.nc4 amsua_n19.tlapse.txt"
 for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.$file $COM_ATMOS_ANALYSIS_PREV/$gprefix.$file
 done
 
 # Link member atmospheric background on tiles and atmf006
@@ -104,10 +104,10 @@ for imem in $(seq 1 $NMEM_ENS); do
 done
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
-elif [ $machine = 'ORION' ]; then
-    sbatch --ntasks=1 --account=$ACCOUNT --qos=batch --partition=orion --time=00:10:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE
 fi

test/atm/global-workflow/jjob_ens_run.sh

@@ -36,22 +36,22 @@ export PYTHONPATH
 machine=$(echo `grep 'machine=' $EXPDIR/config.base | cut -d"=" -f2` | tr -d '"')
 
 # Set NETCDF and UTILROOT variables (used in config.base)
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
     NETCDF=$( which ncdump )
     export NETCDF
     export UTILROOT="/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/intel-18.0.5.274/prod_util/1.2.2"
-elif [ $machine = 'ORION' ]; then
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     ncdump=$( which ncdump )
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2
 fi
 
 # Execute j-job
-if [ $machine = 'HERA' ]; then
+if [[ $machine = 'HERA' ]]; then
+    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
+elif [[ $machine = 'ORION' || $machine = 'HERCULES' ]]; then
     sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
-elif [ $machine = 'ORION' ]; then
-    sbatch --nodes=1 --ntasks=36 --account=$ACCOUNT --qos=batch --partition=orion --time=00:30:00 --export=ALL --wait ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 else
     ${HOMEgfs}/jobs/JGLOBAL_ATMENS_ANALYSIS_RUN
 fi