Merge branch 'NOAA-EMC:develop' into feature/intel2022

RussTreadon-NOAA · Jul 1, 2023 · eb14906 · eb14906
2 parents 3840fda + 333ae16
commit eb14906
Show file tree

Hide file tree

Showing 122 changed files with 6,156 additions and 4,857 deletions.
diff --git a/ci/spack.yaml b/ci/spack.yaml
@@ -18,7 +18,7 @@ spack:
   - [email protected]
   - [email protected]
   - [email protected]
-  - crtm@2.3.0
+  - crtm@2.4.0
   - [email protected]
   view: true
   concretizer:

diff --git a/fix b/fix
diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua
@@ -14,7 +14,7 @@ local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1"
 local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
 local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0"
 local ncio_ver=os.getenv("ncio_ver") or "1.1.2"
-local crtm_ver=os.getenv("crtm_ver") or "2.3.0"
+local crtm_ver=os.getenv("crtm_ver") or "2.4.0"
 local ncdiag_ver=os.getenv("ncdiag_ver") or "1.0.0"
 
 load(pathJoin("netcdf", netcdf_ver))

diff --git a/modulefiles/gsi_gaea b/modulefiles/gsi_gaea
@@ -52,9 +52,6 @@ module load sigio-intel-sandybridge/2.0.1
 module load sp-intel-sandybridge/2.0.2
 module load w3nco-intel-sandybridge/2.0.6
 module load w3emc-intel-sandybridge/2.2.0
-module load crtm-intel/2.2.4
-#setenv CRTM_INC /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/crtm/v2.2.4/intel/include/crtm_v2.2.4
-#setenv CRTM_LIB /lustre/f1/pdata/ncep_shared/NCEPLIBS/lib/crtm/v2.2.4/intel/libcrtm_v2.2.4.a 
 module load bacio-intel-sandybridge/2.0.2
 setenv CRAYOS_VERSION $::env(CRAYPE_VERSION)
 #setenv CRAYOS_VERSION ${CRAYPE_VERSION}

diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua
@@ -20,6 +20,6 @@ load(pathJoin("prod_util", prod_util_ver))
 
 pushenv("MKLROOT", "/apps/oneapi/mkl/2022.0.2")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Hera with GNU Compilers")
diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua
@@ -26,6 +26,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/scratch1/NCEPDEV/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Hera with Intel Compilers")
diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua
@@ -1,26 +1,32 @@
 help([[
 ]])
 
-load("cmake/3.20.1")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack")
 
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
+local hpc_ver=os.getenv("hpc_ver") or "1.2.0"
+local hpc_intel_ver=os.getenv("hpc_intel_ver") or "18.0.5.274"
+local hpc_impi_ver=os.getenv("hpc_impi_ver") or "2018.4.274"
+local cmake_ver=os.getenv("cmake_ver") or "3.20.1"
+local anaconda_ver=os.getenv("anaconda_ver") or "5.3.1"
+local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
 
-load("anaconda/5.3.1")
+load(pathJoin("hpc", hpc_ver))
+load(pathJoin("hpc-intel", hpc_intel_ver))
+load(pathJoin("hpc-impi", hpc_impi_ver))
+load(pathJoin("cmake", cmake_ver))
 
-prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/nwprod/hpc-stack/libs/modulefiles/stack")
+prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
 
-load("hpc/1.1.0")
-load("hpc-intel/18.0.5.274")
-load("hpc-impi/2018.4.274")
+load(pathJoin("anaconda", anaconda_ver))
 
 load("gsi_common")
 
-local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2"
 load(pathJoin("prod_util", prod_util_ver))
 
 pushenv("CFLAGS", "-axSSE4.2,AVX,CORE-AVX2")
 pushenv("FFLAGS", "-axSSE4.2,AVX,CORE-AVX2")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20221128")
+
+pushenv("GSI_BINARY_SOURCE_DIR", "/mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Jet with Intel Compilers")
diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua
@@ -23,6 +23,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-xHOST")
 pushenv("FFLAGS", "-xHOST")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/work/noaa/global/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on Orion with Intel Compilers")
diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua
@@ -23,6 +23,6 @@ load(pathJoin("prod_util", prod_util_ver))
 pushenv("CFLAGS", "-march=ivybridge")
 pushenv("FFLAGS", "-march=ivybridge")
 
-pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/data/prod/glopara/fix/gsi/20230601")
 
 whatis("Description: GSI environment on S4 with Intel Compilers")
diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua
@@ -19,16 +19,7 @@ load(pathJoin("python", python_ver))
 load(pathJoin("prod_util", prod_util_ver))
 
 load("gsi_common")
-unload("ncio")
-unload("ncdiag")
 
-pushenv("HPC_OPT", "/apps/ops/para/libs")
-prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304")
-prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7")
-
-load("ncio/1.1.2")
-load("ncdiag/1.0.0")
-
-pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20221128")
+pushenv("GSI_BINARY_SOURCE_DIR", "/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/gsi/20230601")
 
 whatis("Description: GSI environment on WCOSS2")
diff --git a/regression/global_3dvar.sh b/regression/global_3dvar.sh
@@ -294,7 +294,7 @@ for type in $listdiag; do
    date=`echo $diag_file | cut -d'.' -f2`
    $UNCOMPRESS $diag_file
    fnameanl=$(echo $fname|sed 's/_ges//g')
-   mv $fname.$date $fnameanl
+   mv ${fname}.${date} $fnameanl
 done
 
 # Run GSI

diff --git a/regression/netcdf_fv3_regional.sh b/regression/netcdf_fv3_regional.sh
@@ -7,15 +7,6 @@ set -x
 # Set experiment name
 exp=$jobname
 
-#TM=00
-#TM2=03
-#tmmark=tm${TM}
-
-
-# Set path/file for gsi executable
-#gsiexec=/meso/save/Wanshu.Wu/Code/trunk/trunk_40320/src/global_gsi_org
-#gsiexec=/da/save/Michael.Lueken/trunk/src/global_gsi.x
-
 # Set runtime and save directories
 tmpdir=$tmpdir/tmpreg_netcdf_fv3_regional/${exp}
 savdir=$savdir/outreg_netcdf_fv3_regional/${exp}

diff --git a/regression/regression_param.sh b/regression/regression_param.sh
@@ -53,8 +53,8 @@ case $regtest in
            topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "Jet" ]]; then
-           topts[1]="0:50:00" ; popts[1]="12/3/" ; ropts[1]="/1"
-           topts[2]="0:50:00" ; popts[2]="12/9/" ; ropts[2]="/2"
+           topts[1]="0:15:00" ; popts[1]="12/5/" ; ropts[1]="/1"
+           topts[2]="0:15:00" ; popts[2]="12/9/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
            topts[1]="0:30:00" ; popts[1]="16/2/" ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="16/4/" ; ropts[2]="/2"
@@ -123,8 +123,8 @@ case $regtest in
            topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
            topts[2]="0:10:00" ; popts[2]="12/12/" ; ropts[2]="/2"
         elif [[ "$machine" = "Jet" ]]; then
-           topts[1]="0:35:00" ; popts[1]="6/8/" ; ropts[1]="/1"
-           topts[2]="0:35:00" ; popts[2]="6/10/" ; ropts[2]="/2"
+           topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1"
+           topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2"
         elif [[ "$machine" = "Discover" ]]; then
            topts[1]="0:30:00" ; popts[1]="48/2"  ; ropts[1]="/1"
            topts[2]="0:30:00" ; popts[2]="60/3"  ; ropts[2]="/2"
@@ -153,8 +153,8 @@ case $regtest in
             topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "Jet" ]]; then
-            topts[1]="0:15:00" ; popts[1]="4/4/"  ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="6/6/"  ; ropts[2]="/1"
+            topts[1]="0:15:00" ; popts[1]="20/1/"  ; ropts[1]="/1"
+            topts[2]="0:15:00" ; popts[2]="20/2/"  ; ropts[2]="/1"
         elif [[ "$machine" = "wcoss2" ]]; then
            topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1"
            topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1"
@@ -255,8 +255,8 @@ case $regtest in
             topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
             topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "Jet" ]]; then
-            topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1"
-            topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2"
+            topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1"
+            topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2"
         elif [[ "$machine" = "Cheyenne" ]]; then
             topts[1]="0:15:00" ; popts[1]="16/2/" ; ropts[1]="/1"
             topts[2]="0:15:00" ; popts[2]="16/4/" ; ropts[2]="/2"

diff --git a/regression/regression_var.sh b/regression/regression_var.sh
@@ -33,7 +33,7 @@ if [[ -d /glade ]]; then # Cheyenne
   export machine="Cheyenne"
 elif [[ -d /scratch1 ]]; then # Hera
   export machine="Hera"
-elif [[ -d /jetmon ]]; then # Jet
+elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs1 ]]; then # Jet
   export machine="Jet"
 elif [[ -d /discover ]]; then # NCCS Discover
   export machine="Discover"
@@ -58,7 +58,6 @@ case $machine in
     fi
     export ptmp="/glade/scratch/$LOGNAME/$ptmpName"
 
-    export fixcrtm="/glade/p/ral/jntp/tools/crtm/2.2.3/fix_update"
     export casesdir="/glade/p/ral/jntp/tools/CASES"
 
     export check_resource="no"
@@ -99,7 +98,6 @@ case $machine in
       fi
       export ptmp="${ptmp:-/work/noaa/stmp/$LOGNAME/$ptmpName}"
 
-      export fixcrtm=${CRTM_FIX:-/apps/contrib/NCEPLIBS/orion/fix/crtm_v2.3.0}
       export casesdir="/work/noaa/da/rtreadon/CASES/regtest"
 
       export check_resource="no"
@@ -124,7 +122,6 @@ case $machine in
 
     export ptmp="${ptmp:-/scratch1/NCEPDEV/stmp2/$LOGNAME/$ptmpName}"
 
-##  export fixcrtm="${CRTM_FIX:-/scratch1/NCEPDEV/da/Michael.Lueken/CRTM_REL-2.2.3/crtm_v2.2.3/fix_update}"
     export casesdir="/scratch1/NCEPDEV/da/Russ.Treadon/CASES/regtest"
 
     export check_resource="no"
@@ -138,19 +135,16 @@ case $machine in
 
     export noscrub=/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/noscrub
     export ptmp=/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/ptmp
-    export fixcrtm="/lfs1/NESDIS/nesdis-rdo2/David.Huber/save/CRTM_REL-2.2.3/crtm_v2.2.3/fix_update"
-    export casesdir="/lfs1/NESDIS/nesdis-rdo2/David.Huber/save/CASES"
+    export casesdir="/lfs1/NESDIS/nesdis-rdo2/David.Huber/save/CASES/regtest"
     export check_resource="no"
     export accnt="nesdis-rdo2"
 
     export group="global"
     export queue="batch"
     if [[ "$cmaketest" = "false" ]]; then
-      export basedir="/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/gsi"
+      export basedir="/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/save/git/gsi"
     fi
 
-    export ptmp="/lfs1/NESDIS/nesdis-rdo2/$LOGNAME/ptmp/$ptmpName"
-
     #  On Jet, there are no scrubbers to remove old contents from stmp* directories.
     #  After completion of regression tests, will remove the regression test subdirecories
     export clean=".true."
@@ -163,7 +157,6 @@ case $machine in
     export ptmp=$basedir
     export ptmp=$basedir
     export noscrub=$basedir
-    export fixcrtm="/discover/nobackup/projects/gmao/share/gmao_ops/fvInput_4dvar/gsi/etc/fix_ncep20170329/REL-2.2.3-r60152_local-rev_1/CRTM_Coeffs/$endianness"
     export casesdir="/discover/nobackup/projects/gmao/obsdev/wrmccart/NCEP_regression/CASES"
     export check_resource="no"
     export accnt="g0613"
@@ -176,18 +169,6 @@ case $machine in
   ;;
 esac
 
-if [[ "$cmaketest" = "false" ]]; then
-  export builddir=$noscrub/build
-  export gsisrc="$basedir/$updat/src"
-  export gsiexec_updat="$gsisrc/global_gsi.x"
-  export gsiexec_contrl="$basedir/$contrl/src/global_gsi.x"
-  export enkfexec_updat="$gsisrc/enkf/global_enkf.x"
-  export enkfexec_contrl="$basedir/$contrl/src/enkf/global_enkf.x"
-  export fixgsi="$basedir/$updat/fix"
-  export scripts="$basedir/$updat/regression"
-  export ush="$basedir/$updat/ush"
-fi
-
 # We are dealing with *which* endian files
 export endianness="Big_Endian"
 

diff --git a/src/enkf/controlvec.f90 b/src/enkf/controlvec.f90
@@ -131,7 +131,7 @@ subroutine init_controlvec()
       cvars3d(nc3d) = trim(adjustl(var))
       clevels(nc3d) = ilev + clevels(nc3d-1)
    else 
-      if (nproc .eq. 0) print *,'Error: only ', nlevs, ' and ', nlevs+1,' number of levels is supported in current version, got ',ilev
+      if (nproc .eq. 0) print *,'Error controlvec: only ', nlevs, ' and ', nlevs+1,' number of levels is supported in current version, got ',ilev
       call stop2(503)
    endif
 enddo
@@ -212,7 +212,10 @@ subroutine read_control()
 ! read in whole control vector on i/o procs - keep in memory 
 ! (needed in write_ensemble)
 allocate(grdin(npts,ncdim,nbackgrounds,nanals_per_iotask))
-allocate(qsat(npts,nlevs,nbackgrounds,nanals_per_iotask))
+! if only updating the sfc fields, qsat will not be calculated in readgriddata
+! only allocate if needed.
+q_ind = getindex(cvars3d, 'q')
+if (q_ind > 0)  allocate(qsat(npts,nlevs,nbackgrounds,nanals_per_iotask))
 if (paranc) then
    if (nproc == 0) t1 = mpi_wtime()
    call readgriddata_pnc(cvars3d,cvars2d,nc3d,nc2d,clevels,ncdim,nbackgrounds, &
@@ -225,7 +228,8 @@ subroutine read_control()
            fgfileprefixes,fgsfcfileprefixes,reducedgrid,grdin,qsat)
    end if
    !print *,'min/max qsat',nanal,'=',minval(qsat),maxval(qsat)
-   if (use_qsatensmean) then
+   q_ind = getindex(cvars3d, 'q')
+   if (use_qsatensmean .and. q_ind>0 ) then
        allocate(qsatmean(npts,nlevs,nbackgrounds))
        allocate(qsat_tmp(npts))
        ! compute ensemble mean qsat
@@ -257,7 +261,6 @@ subroutine read_control()
    !   print *,'min/max qsatmean proc',nproc,'=',&
    !            minval(qsatmean(:,:,nbackgrounds/2+1)),maxval(qsatmean(:,:,nbackgrounds/2+1))
    !endif
-   q_ind = getindex(cvars3d, 'q')
    if (pseudo_rh .and. q_ind > 0) then
       if (use_qsatensmean) then
          do ne=1,nanals_per_iotask