diff --git a/env/GAEA.env b/env/GAEA.env index 6809a9b186..be5e9f0ca7 100755 --- a/env/GAEA.env +++ b/env/GAEA.env @@ -34,12 +34,42 @@ else exit 2 fi -if [[ "${step}" = "waveinit" ]]; then +if [[ "${step}" = "prep" ]]; then - export CFP_MP="YES" - if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi - export wavempexec=${launcher} - export wave_mpmd=${mpmd_opt} + export POE="NO" + export BACK="NO" + export sys_tp="GAEA" + export launcher_PREP="srun" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}" elif [[ "${step}" = "fcst" ]]; then @@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "upp" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}" + elif [[ "${step}" = "atmos_products" ]]; then - export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + export USE_CFP="YES" # Use MPMD for downstream product generation on Gaea elif [[ "${step}" = "oceanice_products" ]]; then diff --git a/modulefiles/module_base.gaea.lua b/modulefiles/module_base.gaea.lua index 55ad6b0c34..b08e79c274 100644 --- a/modulefiles/module_base.gaea.lua +++ b/modulefiles/module_base.gaea.lua @@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None"))) load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None"))) load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None"))) load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None"))) +load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None"))) load(pathJoin("nco", (os.getenv("nco_ver") or "None"))) load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None"))) @@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None"))) load(pathJoin("bufr", (os.getenv("bufr_ver") or "None"))) load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None"))) load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None"))) +load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None"))) load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None"))) load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None"))) load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None"))) @@ -36,4 +38,13 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None"))) setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) +--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles")) +load(pathJoin("prepobs", "1.1.0")) + +prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None"))) + + whatis("Description: GFS run setup environment") diff --git a/versions/run.gaea.ver b/versions/run.gaea.ver index b92fe8c1db..c3aceb445d 100644 --- a/versions/run.gaea.ver +++ b/versions/run.gaea.ver @@ -2,5 +2,7 @@ export stack_intel_ver=2023.1.0 export stack_cray_mpich_ver=8.1.25 export spack_env=gsi-addon-dev +export perl_ver=5.38.2 + source "${HOMEgfs:-}/versions/run.spack.ver" export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"