diff --git a/cmake/configure_gaea.intel.cmake b/cmake/configure_gaeac5.intel.cmake similarity index 100% rename from cmake/configure_gaea.intel.cmake rename to cmake/configure_gaeac5.intel.cmake diff --git a/cmake/configure_gaea.intelllvm.cmake b/cmake/configure_gaeac5.intelllvm.cmake similarity index 100% rename from cmake/configure_gaea.intelllvm.cmake rename to cmake/configure_gaeac5.intelllvm.cmake diff --git a/cmake/configure_gaeac6.intel.cmake b/cmake/configure_gaeac6.intel.cmake new file mode 100644 index 0000000000..6037c6379f --- /dev/null +++ b/cmake/configure_gaeac6.intel.cmake @@ -0,0 +1,4 @@ +set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE) +set(MOM6_Extra_FORTRAN_FLAGS "-xsse2") +set(HYCOM_Extra_FORTRAN_FLAGS "-xSSE4.2") +set(HYCOM_Extra_C_FLAGS "-xSSE4.2") \ No newline at end of file diff --git a/cmake/configure_gaeac6.intelllvm.cmake b/cmake/configure_gaeac6.intelllvm.cmake new file mode 100644 index 0000000000..42e6486231 --- /dev/null +++ b/cmake/configure_gaeac6.intelllvm.cmake @@ -0,0 +1,2 @@ +set(PARALLEL_NETCDF ON CACHE BOOL "Enable parallel NetCDF" FORCE) +set(AVX2 OFF CACHE BOOL "Enable AVX2 instruction set" FORCE) diff --git a/modulefiles/ufs_gaea.intel.lua b/modulefiles/ufs_gaeac5.intel.lua similarity index 88% rename from modulefiles/ufs_gaea.intel.lua rename to modulefiles/ufs_gaeac5.intel.lua index 834c8fc41f..18179b8d4b 100644 --- a/modulefiles/ufs_gaea.intel.lua +++ b/modulefiles/ufs_gaeac5.intel.lua @@ -1,9 +1,9 @@ help([[ This module loads libraries required for building and running UFS Weather Model - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. + on the NOAA RDHPC machine Gaea C5 using Intel-2023.2.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===]) +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5]===]) prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") @@ -30,4 +30,4 @@ unload("cray-libsci") setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") -setenv("CMAKE_Platform","gaea.intel") +setenv("CMAKE_Platform","gaeac5.intel") diff --git a/modulefiles/ufs_gaea.intelllvm.lua b/modulefiles/ufs_gaeac5.intelllvm.lua similarity index 84% rename from modulefiles/ufs_gaea.intelllvm.lua rename to modulefiles/ufs_gaeac5.intelllvm.lua index 01d8585454..f6b9abf904 100644 --- a/modulefiles/ufs_gaea.intelllvm.lua +++ b/modulefiles/ufs_gaeac5.intelllvm.lua @@ -1,9 +1,9 @@ help([[ This module loads libraries required for building and running UFS Weather Model - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. + on the NOAA RDHPC machine Gaea C5 using Intel-2023.2.0. ]]) -whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea ]===]) +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C5]===]) prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") @@ -27,7 +27,8 @@ load(pathJoin("nccmp", nccmp_ver)) unload("darshan-runtime") unload("cray-libsci") -load("intel-classic/2023.2.0") +unload("intel-classic/2023.2.0") +load("intel-oneapi/2023.2.0") setenv("I_MPI_CC", "icx") setenv("I_MPI_CXX", "icpx") @@ -36,4 +37,4 @@ setenv("I_MPI_F90", "ifx") setenv("CC","cc") setenv("CXX","CC") setenv("FC","ftn") -setenv("CMAKE_Platform","gaea.intelllvm") +setenv("CMAKE_Platform","gaeac5.intelllvm") diff --git a/modulefiles/ufs_gaeac6.intel.lua b/modulefiles/ufs_gaeac6.intel.lua new file mode 100644 index 0000000000..ce59797288 --- /dev/null +++ b/modulefiles/ufs_gaeac6.intel.lua @@ -0,0 +1,33 @@ +help([[ + This module loads libraries required for building and running UFS Weather Model + on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0. +]]) + +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6]===]) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("ufs_common") + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +unload("darshan-runtime") +unload("cray-libsci") + +setenv("CC","cc") +setenv("CXX","CC") +setenv("FC","ftn") +setenv("CMAKE_Platform","gaeac6.intel") diff --git a/modulefiles/ufs_gaeac6.intelllvm.lua b/modulefiles/ufs_gaeac6.intelllvm.lua new file mode 100644 index 0000000000..abf311b4ec --- /dev/null +++ b/modulefiles/ufs_gaeac6.intelllvm.lua @@ -0,0 +1,40 @@ +help([[ + This module loads libraries required for building and running UFS Weather Model + on the NOAA RDHPC machine Gaea C6 using Intel-2023.2.0. +]]) + +whatis([===[Loads libraries needed for building the UFS Weather Model on Gaea C6]===]) + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.29" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("ufs_common") + +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" +load(pathJoin("nccmp", nccmp_ver)) + +unload("darshan-runtime") +unload("cray-libsci") + +unload("intel-classic/2023.2.0") +load("intel-oneapi/2023.2.0") + +setenv("I_MPI_CC", "icx") +setenv("I_MPI_CXX", "icpx") +setenv("I_MPI_F90", "ifx") + +setenv("CC","cc") +setenv("CXX","CC") +setenv("FC","ftn") +setenv("CMAKE_Platform","gaeac6.intelllvm") diff --git a/tests/compile.sh b/tests/compile.sh index dd8f91ced1..2c00304e28 100755 --- a/tests/compile.sh +++ b/tests/compile.sh @@ -61,7 +61,10 @@ case ${MACHINE_ID} in ;; *) # Activate lua environment for gaea c5 - if [[ ${MACHINE_ID} == gaea ]]; then + if [[ ${MACHINE_ID} == gaeac5 ]]; then + module reset + fi + if [[ ${MACHINE_ID} == gaeac6 ]]; then module reset elif [[ ${MACHINE_ID} == hercules ]]; then module purge @@ -97,17 +100,6 @@ SUITES=$(grep -Po "\-DCCPP_SUITES=\K[^ ]*" <<< "${MAKE_OPT}") export SUITES set -ex -# Valid applications -if [[ ${MACHINE_ID} != gaea ]] || [[ ${RT_COMPILER} != intelllvm ]]; then # skip MOM6SOLO on gaea with intelllvm - if [[ "${MAKE_OPT}" == *"-DAPP=S2S"* ]]; then - CMAKE_FLAGS+=" -DMOM6SOLO=ON" - fi - - if [[ "${MAKE_OPT}" == *"-DAPP=NG-GODAS"* ]]; then - CMAKE_FLAGS+=" -DMOM6SOLO=ON" - fi -fi - CMAKE_FLAGS=$(set -e; trim "${CMAKE_FLAGS}") echo "CMAKE_FLAGS = ${CMAKE_FLAGS}" diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 4b46ce6aaf..ff086e88f2 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -300,7 +300,7 @@ elif [[ ${MACHINE_ID} = s4 ]]; then export ICE_tasks_cpl_bmrk=48 export WAV_tasks_cpl_bmrk=80 -elif [[ ${MACHINE_ID} = gaea ]]; then +elif [[ ${MACHINE_ID} = gaeac5 ]]; then export TPN=128 @@ -321,6 +321,26 @@ elif [[ ${MACHINE_ID} = gaea ]]; then export WPG_cpl_atmw_gdas=24 export WAV_tasks_atmw_gdas=264 +elif [[ ${MACHINE_ID} = gaeac6 ]]; then + + export TPN=192 + + export INPES_dflt=3 + export JNPES_dflt=8 + export INPES_thrd=3 + export JNPES_thrd=4 + export INPES_c384=6 + export JNPES_c384=8 + export THRD_c384=1 + export INPES_c768=8 + export JNPES_c768=16 + export THRD_c768=2 + + export THRD_cpl_atmw_gdas=3 + export INPES_cpl_atmw_gdas=6 + export JNPES_cpl_atmw_gdas=8 + export WPG_cpl_atmw_gdas=24 + export WAV_tasks_atmw_gdas=264 elif [[ ${MACHINE_ID} = derecho ]]; then export TPN=128 diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 0bd0535d8a..99419f3555 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -21,8 +21,10 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58 + gaea6[1-8]) MACHINE_ID=gaeac6 ;; ### gaea61-68 + gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68 hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 @@ -94,9 +96,12 @@ elif [[ -d /work ]]; then else MACHINE_ID=orion fi -elif [[ -d /gpfs && -d /ncrc ]]; then - # We are on GAEA. - MACHINE_ID=gaea +elif [[ -d /gpfs/f5 && -d /ncrc ]]; then + # We are on GAEA C5. + MACHINE_ID=gaeac5 +elif [[ -d /gpfs/f6 && -d /ncrc ]]; then + # We are on GAEA C6. + MACHINE_ID=gaeac6 elif [[ -d /data/prod ]]; then # We are on SSEC's S4 MACHINE_ID=s4 diff --git a/tests/fv3_conf/compile_slurm.IN_gaea b/tests/fv3_conf/compile_slurm.IN_gaeac5 similarity index 100% rename from tests/fv3_conf/compile_slurm.IN_gaea rename to tests/fv3_conf/compile_slurm.IN_gaeac5 diff --git a/tests/fv3_conf/compile_slurm.IN_gaeac6 b/tests/fv3_conf/compile_slurm.IN_gaeac6 new file mode 100644 index 0000000000..9f5bf9c304 --- /dev/null +++ b/tests/fv3_conf/compile_slurm.IN_gaeac6 @@ -0,0 +1,22 @@ +#!/bin/bash -l +#SBATCH -e err +#SBATCH -o out +#SBATCH --account=@[ACCNR] +##SBATCH --qos=@[QUEUE] +#SBATCH --clusters=es +#SBATCH --partition=eslogin_c6 +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=8 +#SBATCH --mem-per-cpu=4G +#SBATCH --time=180 +#SBATCH --job-name="@[JBNME]" + +set -eux + +echo -n " $( date +%s )," > job_timestamp.txt +echo "Compile started: " `date` + +@[PATHRT]/compile.sh @[MACHINE_ID] "@[MAKE_OPT]" @[COMPILE_ID] @[RT_COMPILER] + +echo "Compile ended: " `date` +echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/fv3_conf/fv3_slurm.IN_gaea b/tests/fv3_conf/fv3_slurm.IN_gaeac5 similarity index 100% rename from tests/fv3_conf/fv3_slurm.IN_gaea rename to tests/fv3_conf/fv3_slurm.IN_gaeac5 diff --git a/tests/fv3_conf/fv3_slurm.IN_gaeac6 b/tests/fv3_conf/fv3_slurm.IN_gaeac6 new file mode 100644 index 0000000000..9511554fb7 --- /dev/null +++ b/tests/fv3_conf/fv3_slurm.IN_gaeac6 @@ -0,0 +1,49 @@ +#!/bin/bash -l +#SBATCH -e err +#SBATCH -o out +#SBATCH --job-name="@[JBNME]" +#SBATCH --account=@[ACCNR] +#SBATCH --qos=@[QUEUE] +#SBATCH --clusters=c6 +#SBATCH --partition=batch +#SBATCH --nodes=@[NODES] +#SBATCH --ntasks-per-node=@[TPN] +#SBATCH --time=@[WLCLK] + +set -eux +echo -n " $( date +%s )," > job_timestamp.txt + +set +x +MACHINE_ID=gaeac6 +source ./module-setup.sh +module use --prepend $PWD/modulefiles +module load modules.fv3 +module list +set -x + +echo "Model started: " `date` + +export OMP_NUM_THREADS=@[THRD] +export OMP_STACKSIZE=1024M +export NC_BLKSZ=1M +export ESMF_RUNTIME_PROFILE=ON +export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY" +export FI_VERBS_PREFER_XRC=0 +export FI_CXI_RX_MATCH_MODE=hybrid +export COMEX_EAGER_THRESHOLD=65536 +export FI_CXI_RDZV_THRESHOLD=65536 + +# Avoid job errors because of filesystem synchronization delays +sync && sleep 1 + +# This "if" block is part of the rt.sh self-tests in error-test.conf. It emulates the model failing to run. +if [ "${JOB_SHOULD_FAIL:-NO}" = WHEN_RUNNING ] ; then + echo "The job should abort now, with exit status 1." 1>&2 + echo "If error checking is working, the metascheduler should mark the job as failed." 1>&2 + false +fi + +srun --label -n @[TASKS] ./fv3.exe + +echo "Model ended: " `date` +echo -n " $( date +%s )," >> job_timestamp.txt diff --git a/tests/logs/RegressionTests_gaea.log b/tests/logs/RegressionTests_gaeac5.log similarity index 100% rename from tests/logs/RegressionTests_gaea.log rename to tests/logs/RegressionTests_gaeac5.log diff --git a/tests/logs/RegressionTests_gaeac6.log b/tests/logs/RegressionTests_gaeac6.log new file mode 100644 index 0000000000..2b68a558ab --- /dev/null +++ b/tests/logs/RegressionTests_gaeac6.log @@ -0,0 +1,68 @@ +====START OF GAEAC6 REGRESSION TESTING LOG==== + +UFSWM hash used in testing: +2ccc549348da37aac51ab44482174dff2bb2912d + +Submodule hashes used in testing: + 37cbb7d6840ae7515a9a8f0dfd4d89461b3396d1 AQM (v0.2.0-37-g37cbb7d) + be5d28fd1b60522e6fc98aefeead20e6aac3530b AQM/src/model/CMAQ (CMAQv5.2.1_07Feb2018-198-gbe5d28fd1) + 1f9eaaa142c8b07ed6b788c9f44ea02cc86d0bae CDEPS-interface/CDEPS (cdeps0.4.17-42-g1f9eaaa) + 635d9a100a736bd8d14ad091e879d5da6e4eb2bd CICE-interface/CICE (CICE6.0.0-373-g635d9a1) + 4c87095256c1c599c3ccaa857a95744158751a60 CICE-interface/CICE/icepack (Icepack1.1.0-191-g4c87095) + dc977bcadd1ade1a528dee75f1ad45e8bd80ca0a CMEPS-interface/CMEPS (cmeps_v0.4.1-2310-gdc977bc) + cabd7753ae17f7bfcc6dad56daf10868aa51c3f4 CMakeModules (v1.0.0-28-gcabd775) + a9364591091c836984a40107729720705847c195 FV3 (heads/develop) + ac3055eff06099d61cd65e18bc4f0353ffd83f46 FV3/atmos_cubed_sphere (201912_public_release-405-gac3055e) + 0f8232724975c13289cad390c9a71fa2c6a9bff4 FV3/ccpp/framework (2024-07-11-dev) + b6c433354394bd8ed5e46692a81149441ff4ae38 FV3/ccpp/physics (EP4-873-gb6c43335) + 74a0e098b2163425e4b5466c2dfcf8ae26d560a5 FV3/ccpp/physics/physics/Radiation/RRTMGP/rte-rrtmgp (v1.6) + 81b38a88d860ce7e34e8507c2246151a54d96a39 FV3/upp (upp_v10.2.0-218-g81b38a88) +-1ba8270870947b583cd51bc72ff8960f4c1fb36e FV3/upp/sorc/libIFI.fd +-567edcc94bc418d0dcd6cdaafed448eeb5aab570 FV3/upp/sorc/ncep_post.fd/post_gtg.fd + 041422934cae1570f2f0e67239d5d89f11c6e1b7 GOCART (sdr_v2.1.2.6-119-g0414229) + bcf7777bb037ae2feb2a8a8ac51aacb3511b52d9 HYCOM-interface/HYCOM (2.3.00-122-gbcf7777) + 5e0c21f64fa5b20efc8f29f8709766e1e6793a79 MOM6-interface/MOM6 (dev/master/repository_split_2014.10.10-10230-g5e0c21f64) + 9423197f894112edfcb1502245f7d7b873d551f9 MOM6-interface/MOM6/pkg/CVMix-src (9423197) + 29e64d652786e1d076a05128c920f394202bfe10 MOM6-interface/MOM6/pkg/GSW-Fortran (29e64d6) + 3ac32f0db7a2a97d930f44fa5f060c983ff31ee8 NOAHMP-interface/noahmp (v3.7.1-436-g3ac32f0) + 7f548c795a348bbb0fe4967dd25692c79036dc73 WW3 (6.07.1-346-g7f548c79) + 05cad173feeb598431e3ef5f17c2df6562c8d101 fire_behavior (v0.2.0-1-g05cad17) + fad2fe9f42f6b7f744b128b4a2a9433f91e4296f stochastic_physics (ufs-v2.0.0-219-gfad2fe9) + + +NOTES: +[Times](Memory) are at the end of each compile/test in format [MM:SS](Size). +The first time is for the full script (prep+run+finalize). +The second time is specifically for the run phase. +Times/Memory will be empty for failed tests. + +BASELINE DIRECTORY: /gpfs/f6/drsa-fire2/world-shared/Brian.Curtis/NEMSfv3gfs/develop-20240909 +COMPARISON DIRECTORY: /gpfs/f6/drsa-fire2/scratch/Brian.Curtis/RT_RUNDIRS/Brian.Curtis/FV3_RT/rt_2186049 + +RT.SH OPTIONS USED: +* (-a) - HPC PROJECT ACCOUNT: drsa-fire2 +* (-c) - CREATE NEW BASELINES +* (-n) - RUN SINGLE TEST: cpld_control_p8 +* (-e) - USE ECFLOW + +PASS -- COMPILE 's2swa_intel' [09:12, 08:08] ( 6 warnings 10 remarks ) +FAILED: TEST TIMED OUT -- TEST 'cpld_control_p8_intel' [, ]( MB) + +SYNOPSIS: +Starting Date/Time: 20241002 11:56:43 +Ending Date/Time: 20241002 13:19:28 +Total Time: 01h:23m:40s +Compiles Completed: 1/1 +Tests Completed: 0/1 +Failed Tests: +* TEST cpld_control_p8_intel: FAILED: TEST TIMED OUT +-- LOG: /gpfs/f6/drsa-fire2/scratch/Brian.Curtis/RT_RUNDIRS/Brian.Curtis/FV3_RT/rt_2186049/cpld_control_p8_intel/err + +NOTES: +A file 'test_changes.list' was generated with list of all failed tests. +You can use './rt.sh -c -b test_changes.list' to create baselines for the failed tests. +If you are using this log as a pull request verification, please commit 'test_changes.list'. + +Result: FAILURE + +====END OF GAEAC6 REGRESSION TESTING LOG==== diff --git a/tests/module-setup.sh b/tests/module-setup.sh index 5bbba3e465..d39bbeb95c 100755 --- a/tests/module-setup.sh +++ b/tests/module-setup.sh @@ -65,7 +65,7 @@ elif [[ ${MACHINE_ID} = frontera ]] ; then fi module purge -elif [[ ${MACHINE_ID} = gaea ]] ; then +elif [[ ${MACHINE_ID} = gaeac5 ]] ; then # We are on GAEA if ( ! eval module help > /dev/null 2>&1 ) ; then # We cannot simply load the module command. The GAEA @@ -76,7 +76,12 @@ elif [[ ${MACHINE_ID} = gaea ]] ; then source /etc/profile fi module reset - +elif [[ ${MACHINE_ID} = gaeac6 ]]; then + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /opt/cray/pe/lmod/lmod/init/bash + fi + module reset + elif [[ ${MACHINE_ID} = expanse ]]; then # We are on SDSC Expanse if ( ! eval module help > /dev/null 2>&1 ) ; then diff --git a/tests/rt.sh b/tests/rt.sh index 672c876e53..6fed829859 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -709,8 +709,8 @@ case ${MACHINE_ID} in PTMP="/lfs/h2/emc/ptmp" SCHEDULER="pbs" ;; - gaea) - echo "rt.sh: Setting up gaea..." + gaeac5) + echo "rt.sh: Setting up gaea c5..." if [[ "${ROCOTO:-false}" == true ]] ; then module use /ncrc/proj/epic/rocoto/modulefiles module load rocoto @@ -738,6 +738,37 @@ case ${MACHINE_ID} in STMP=${STMP:-${dprefix}/RT_BASELINE} PTMP=${PTMP:-${dprefix}/RT_RUNDIRS} + SCHEDULER="slurm" + ;; + gaeac6) + echo "rt.sh: Setting up gaea c6..." + if [[ "${ROCOTO:-false}" == true ]] ; then + module use /ncrc/proj/epic/c6/modulefiles + module load rocoto/1.3.7 + ROCOTO_SCHEDULER="slurm" + fi + + export LD_PRELOAD=/usr/lib64/libstdc++.so.6 + module use /ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core + module load stack-intel/2023.2.0 + module load cray-mpich/8.1.29 + module load python/3.10.13 + if [[ "${ECFLOW:-false}" == true ]] ; then + module use /ncrc/proj/epic/spack-stack/modulefiles + module load ecflow/5.8.4 + ECF_HOST=$(hostname) + ECF_PORT=$(( $(id -u) + 1500 )) + export ECF_PORT ECF_HOST + fi + + DISKNM=/gpfs/f6/bil-fire8/world-shared/role.epic/UFS-WM_RT + QUEUE=normal + COMPILE_QUEUE=normal + PARTITION=c6 + dprefix=${dprefix:-/gpfs/f6/${ACCNR}/proj-shared/${USER}} + STMP=${STMP:-${dprefix}/RT_BASELINE} + PTMP=${PTMP:-${dprefix}/RT_RUNDIRS} + SCHEDULER="slurm" ;; hera) diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index ecf4259421..6e43e1e914 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -312,7 +312,10 @@ rocoto_create_compile_task() { if [[ ${MACHINE_ID} == s4 ]]; then BUILD_WALLTIME="01:00:00" fi - if [[ ${MACHINE_ID} == gaea ]]; then + if [[ ${MACHINE_ID} == gaeac5 ]]; then + BUILD_WALLTIME="01:00:00" + fi + if [[ ${MACHINE_ID} == gaeac6 ]]; then BUILD_WALLTIME="01:00:00" fi @@ -325,10 +328,15 @@ rocoto_create_compile_task() { ${COMPILE_QUEUE} EOF - if [[ "${MACHINE_ID}" == gaea ]] ; then + if [[ "${MACHINE_ID}" == gaeac5 ]] ; then cat << EOF >> "${ROCOTO_XML}" --clusters=es eslogin_c5 +EOF + elif [[ "${MACHINE_ID}" == gaeac6 ]] ; then + cat << EOF >> "${ROCOTO_XML}" + --clusters=es + eslogin_c6 EOF elif [[ -n "${PARTITION}" || ${MACHINE_ID} != hera ]] ; then cat << EOF >> "${ROCOTO_XML}" @@ -369,7 +377,7 @@ rocoto_create_run_task() { ${ROCOTO_NODESIZE:+${ROCOTO_NODESIZE}} EOF - if [[ "${MACHINE_ID}" == gaea ]] ; then + if [[ "${MACHINE_ID}" == gaeac5 || "${MACHINE_ID}" == gaeac6 ]] ; then cat << EOF >> "${ROCOTO_XML}" --clusters=${PARTITION} --partition=batch diff --git a/tests/run_test.sh b/tests/run_test.sh index 8e8e7c34d3..1724a2b6d7 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -116,11 +116,19 @@ case ${MACHINE_ID} in stampede|expanse|noaacloud) echo "No special nccmp load necessary" ;; - gaea) + gaeac5) module use /ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2023.2.0 stack-cray-mpich/8.1.28 module load nccmp/1.9.0.1 ;; + gaeac6) + module use /ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core + module load stack-intel/2023.2.0 stack-cray-mpich/8.1.29 + module load nccmp/1.9.0.1 + #module use modulefiles + #module load modules.fv3 + #module load gcc-native/12.3 + ;; derecho) module load nccmp ;; @@ -462,7 +470,7 @@ if [[ ${skip_check_results} == false ]]; then else if [[ ${i##*.} == nc* ]] ; then - if [[ " orion hercules hera wcoss2 acorn derecho gaea jet s4 noaacloud " =~ ${MACHINE_ID} ]]; then + if [[ " orion hercules hera wcoss2 acorn derecho gaeac5 gaeac6 jet s4 noaacloud " =~ ${MACHINE_ID} ]]; then printf "USING NCCMP.." >> "${RT_LOG}" printf "USING NCCMP.." if [[ ${CMP_DATAONLY} == false ]]; then diff --git a/tests/tests/atmaero_control_p8 b/tests/tests/atmaero_control_p8 index 6d4012d580..afb6629547 100644 --- a/tests/tests/atmaero_control_p8 +++ b/tests/tests/atmaero_control_p8 @@ -180,3 +180,7 @@ export DOGP_LWSCAT=.true. export DOGP_SGS_CNV=.true. export SATMEDMF=.true. + +if [[ $MACHINE_ID = gaeac6 ]]; then + TPN=96 +fi diff --git a/tests/tests/control_p8.v2.sfc b/tests/tests/control_p8.v2.sfc index d08987ae87..d1afdbc4ca 100644 --- a/tests/tests/control_p8.v2.sfc +++ b/tests/tests/control_p8.v2.sfc @@ -153,6 +153,6 @@ export DOGP_SGS_CNV=.true. export SATMEDMF=.true. -if [[ " hera orion hercules gaea jet " =~ " ${MACHINE_ID} " ]] ; then +if [[ " hera orion hercules gaeac5 gaeac6 jet " =~ " ${MACHINE_ID} " ]] ; then ZSTANDARD_LEVEL=5 fi diff --git a/tests/tests/control_wam b/tests/tests/control_wam index 5018e6208c..d70921938b 100644 --- a/tests/tests/control_wam +++ b/tests/tests/control_wam @@ -106,6 +106,6 @@ export DIAG_TABLE=diag_table_wam export INPUT_NML=wam_v17.nml.IN -if [[ " hera orion hercules gaea jet " =~ " ${MACHINE_ID} " ]] ; then +if [[ " hera orion hercules gaeac5 gaeac6 jet " =~ " ${MACHINE_ID} " ]] ; then ZSTANDARD_LEVEL=5 fi diff --git a/tests/tests/control_wam_debug b/tests/tests/control_wam_debug index 4c1852e2e0..c52d742268 100644 --- a/tests/tests/control_wam_debug +++ b/tests/tests/control_wam_debug @@ -107,6 +107,6 @@ export DIAG_TABLE=diag_table_wam export INPUT_NML=wam_v17.nml.IN -if [[ " hera orion hercules gaea jet " =~ " ${MACHINE_ID} " ]] ; then +if [[ " hera orion hercules gaeac5 gaeac6 jet " =~ " ${MACHINE_ID} " ]] ; then ZSTANDARD_LEVEL=5 fi diff --git a/tests/tests/cpld_control_c192_p8 b/tests/tests/cpld_control_c192_p8 index b70352aee0..3c719ed235 100644 --- a/tests/tests/cpld_control_c192_p8 +++ b/tests/tests/cpld_control_c192_p8 @@ -140,7 +140,7 @@ export FNABSC="'C192.maximum_snow_albedo.tileX.nc'" export FV3_RUN=cpld_control_run.IN -if [[ $MACHINE_ID = derecho ]]; then +if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then TPN=96 fi diff --git a/tests/tests/cpld_control_ciceC_p8 b/tests/tests/cpld_control_ciceC_p8 index 07057d8073..20aeff2e64 100644 --- a/tests/tests/cpld_control_ciceC_p8 +++ b/tests/tests/cpld_control_ciceC_p8 @@ -93,3 +93,7 @@ export N_SPLIT=5 # HR4 GFSv17 GWD update export DO_GWD_OPT_PSL=.true. export DO_GSL_DRAG_SS=.false. + +if [[ $MACHINE_ID = gaeac6 ]]; then + TPN=96 +fi diff --git a/tests/tests/cpld_control_p8 b/tests/tests/cpld_control_p8 index f3cb8da89c..bcb0427037 100644 --- a/tests/tests/cpld_control_p8 +++ b/tests/tests/cpld_control_p8 @@ -85,7 +85,7 @@ export eps_imesh=2.5e-1 export FV3_RUN=cpld_control_run.IN -if [[ $MACHINE_ID = derecho ]]; then +if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then TPN=96 fi diff --git a/tests/tests/cpld_control_p8.v2.sfc b/tests/tests/cpld_control_p8.v2.sfc index bc6e073c61..14bec35bf2 100644 --- a/tests/tests/cpld_control_p8.v2.sfc +++ b/tests/tests/cpld_control_p8.v2.sfc @@ -87,7 +87,7 @@ export eps_imesh=2.5e-1 export FV3_RUN=cpld_control_run.IN export V2_SFC_FILE=true -if [[ $MACHINE_ID = derecho ]]; then +if [[ $MACHINE_ID = derecho ]] || [[ $MACHINE_ID = gaeac6 ]]; then TPN=96 fi diff --git a/tests/tests/cpld_control_p8_faster b/tests/tests/cpld_control_p8_faster index 4e972c16aa..69cb7525a5 100644 --- a/tests/tests/cpld_control_p8_faster +++ b/tests/tests/cpld_control_p8_faster @@ -96,3 +96,7 @@ export DO_GSL_DRAG_LS_BL=.true. export DO_GWD_OPT_PSL=.true. export DO_GSL_DRAG_SS=.false. export DO_GSL_DRAG_TOFD=.true. + +if [[ $MACHINE_ID = gaeac6 ]]; then + TPN=96 +fi diff --git a/tests/tests/cpld_control_p8_mixedmode b/tests/tests/cpld_control_p8_mixedmode index 0d35ba3114..e23f9a0bc2 100644 --- a/tests/tests/cpld_control_p8_mixedmode +++ b/tests/tests/cpld_control_p8_mixedmode @@ -92,3 +92,7 @@ export N_SPLIT=5 # HR4 GFSv17 GWD update export DO_GWD_OPT_PSL=.true. export DO_GSL_DRAG_SS=.false. + +if [[ $MACHINE_ID = gaeac6 ]]; then + TPN=96 +fi diff --git a/tests/tests/cpld_debug_gfsv17 b/tests/tests/cpld_debug_gfsv17 index 63328b4d97..457a73b303 100644 --- a/tests/tests/cpld_debug_gfsv17 +++ b/tests/tests/cpld_debug_gfsv17 @@ -112,7 +112,10 @@ export N_SPLIT=5 if [[ $MACHINE_ID = hera ]] && [[ $RT_COMPILER = gnu ]]; then WLCLK=40 fi -if [[ $MACHINE_ID = gaea ]] && [[ $RT_COMPILER = intelllvm ]]; then +if [[ $MACHINE_ID = gaeac5 ]] && [[ $RT_COMPILER = intelllvm ]]; then + WLCLK=40 +fi +if [[ $MACHINE_ID = gaeac6 ]] && [[ $RT_COMPILER = intelllvm ]]; then WLCLK=40 fi if [[ $MACHINE_ID = orion ]]; then diff --git a/tests/tests/cpld_debug_p8 b/tests/tests/cpld_debug_p8 index 663e21eccf..4c1ca3a24a 100644 --- a/tests/tests/cpld_debug_p8 +++ b/tests/tests/cpld_debug_p8 @@ -81,6 +81,10 @@ if [[ $MACHINE_ID = hera ]] && [[ $RT_COMPILER = gnu ]]; then WLCLK=40 fi +if [[ $MACHINE_ID = gaeac6 ]] ; then + TPN=96 +fi + export K_SPLIT=4 export N_SPLIT=5 diff --git a/tests/tests/cpld_debug_pdlib_p8 b/tests/tests/cpld_debug_pdlib_p8 index e2d0ea77c7..7b8a24e038 100644 --- a/tests/tests/cpld_debug_pdlib_p8 +++ b/tests/tests/cpld_debug_pdlib_p8 @@ -109,7 +109,10 @@ fi if [[ $MACHINE_ID = orion ]]; then WLCLK=50 fi -if [[ $MACHINE_ID = gaea ]]; then +if [[ $MACHINE_ID = gaeac5 ]] && [[ $RT_COMPILER = intelllvm ]]; then + WLCLK=50 +fi +if [[ $MACHINE_ID = gaeac6 ]] && [[ $RT_COMPILER = intelllvm ]]; then WLCLK=50 fi if [[ $MACHINE_ID = wcoss2 ]]; then diff --git a/tests/tests/regional_atmaq b/tests/tests/regional_atmaq index df4603f299..68d0f30739 100644 --- a/tests/tests/regional_atmaq +++ b/tests/tests/regional_atmaq @@ -99,6 +99,6 @@ export CCPP_SUITE=FV3_GFS_v15p2 if [[ $MACHINE_ID = hera ]]; then TPN=30 -elif [[ $MACHINE_ID = gaea ]]; then +elif [[ $MACHINE_ID = gaeac5 ]]; then TPN=16 fi diff --git a/tests/tests/regional_atmaq_debug b/tests/tests/regional_atmaq_debug index 30783d3b52..e45187a264 100644 --- a/tests/tests/regional_atmaq_debug +++ b/tests/tests/regional_atmaq_debug @@ -98,6 +98,8 @@ export CCPP_SUITE=FV3_GFS_v15p2 if [[ $MACHINE_ID = hera ]]; then TPN=30 WLCLK=60 -elif [[ $MACHINE_ID = gaea ]]; then +elif [[ $MACHINE_ID = gaeac5 ]]; then TPN=16 +elif [[ $MACHINE_ID = gaeac6 ]]; then + TPN=128 fi diff --git a/tests/tests/regional_atmaq_faster b/tests/tests/regional_atmaq_faster index 6ccb25375a..17491bc515 100644 --- a/tests/tests/regional_atmaq_faster +++ b/tests/tests/regional_atmaq_faster @@ -93,6 +93,6 @@ export CCPP_SUITE=FV3_GFS_v15p2 if [[ $MACHINE_ID = hera ]]; then TPN=30 -elif [[ $MACHINE_ID = gaea ]]; then +elif [[ $MACHINE_ID = gaeac5 ]]; then TPN=18 fi diff --git a/tests/tests/regional_debug b/tests/tests/regional_debug index f4f6326b90..13f2e328cd 100644 --- a/tests/tests/regional_debug +++ b/tests/tests/regional_debug @@ -41,6 +41,11 @@ if [[ " hera orion hercules jet " =~ " ${MACHINE_ID} " ]] ; then ZSTANDARD_LEVEL=5 fi -if [[ $MACHINE_ID = gaea && $RT_COMPILER = intelllvm ]]; then +if [[ $MACHINE_ID = gaeac5 && $RT_COMPILER = intelllvm ]]; then WLCLK=50 fi + +if [[ $MACHINE_ID = gaeac6 && $RT_COMPILER = intelllvm ]]; then + WLCLK=50 +fi +