From 2e4f4b7671cfec83331ec26de39218543d7b9a6d Mon Sep 17 00:00:00 2001
From: DavidBurrows-NCO <82525974+DavidBurrows-NCO@users.noreply.github.com>
Date: Wed, 4 Sep 2024 13:21:16 -0400
Subject: [PATCH] Add ability to run CI test C96_atm3DVar.yaml to Gaea-C5
 (#2885)

Add the ability to run CI test C96_atm3DVar on Gaea-C5

Resolves #2766
Refs NOAA-EMC/prepobs#32
Refs NOAA-EMC/Fit2Obs#28
---
 env/GAEA.env                     | 47 ++++++++++++++++++++++++++++----
 modulefiles/module_base.gaea.lua | 11 ++++++++
 versions/run.gaea.ver            |  2 ++
 3 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/env/GAEA.env b/env/GAEA.env
index 6809a9b186..be5e9f0ca7 100755
--- a/env/GAEA.env
+++ b/env/GAEA.env
@@ -34,12 +34,42 @@ else
     exit 2
 fi
 
-if [[ "${step}" = "waveinit" ]]; then
+if [[ "${step}" = "prep" ]]; then
 
-    export CFP_MP="YES"
-    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
-    export wavempexec=${launcher}
-    export wave_mpmd=${mpmd_opt}
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="GAEA"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    ntasks_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    ntasks_gausfcanl=${ntasks_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    export NTHREADS_CYCLE=${threads_per_task:-14}
+    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
 
 elif [[ "${step}" = "fcst" ]]; then
 
@@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "upp" ]]; then
+
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"
+
 elif [[ "${step}" = "atmos_products" ]]; then
 
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Gaea
 
 elif [[ "${step}" = "oceanice_products" ]]; then
 
diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua
index 55ad6b0c34..b08e79c274 100644
--- a/modulefiles/module_base.gaea.lua
+++ b/modulefiles/module_base.gaea.lua
@@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
 load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
 load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
 load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None")))
 
 load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
 load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
@@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
 load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
 load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
 load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
@@ -36,4 +38,13 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
+--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles"))
+load(pathJoin("prepobs", "1.1.0"))
+
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+
 whatis("Description: GFS run setup environment")
diff --git a/versions/run.gaea.ver b/versions/run.gaea.ver
index b92fe8c1db..c3aceb445d 100644
--- a/versions/run.gaea.ver
+++ b/versions/run.gaea.ver
@@ -2,5 +2,7 @@ export stack_intel_ver=2023.1.0
 export stack_cray_mpich_ver=8.1.25
 export spack_env=gsi-addon-dev
 
+export perl_ver=5.38.2
+
 source "${HOMEgfs:-}/versions/run.spack.ver"
 export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"