E3SM-Project · jgfouca · Jul 17, 2023 · Jun 5, 2023 · Jun 5, 2023 · Jun 5, 2023
diff --git a/cime_config/machines/cmake_macros/CNL.cmake b/cime_config/machines/cmake_macros/CNL.cmake
diff --git a/cime_config/machines/cmake_macros/Darwin.cmake b/cime_config/machines/cmake_macros/Darwin.cmake
diff --git a/cime_config/machines/cmake_macros/amdclang_crusher.cmake b/cime_config/machines/cmake_macros/amdclang_crusher.cmake
@@ -1,9 +1,19 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+set(NETCDF_PATH "$ENV{NETCDF_DIR}")
+set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
 if (NOT MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
 endif()
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
 set(CRAY_LIBSCI_PREFIX_DIR "$ENV{CRAY_LIBSCI_PREFIX_DIR}")
-#if (COMP_NAME STREQUAL gptl)
-#  string(APPEND CPPDEFS " -DFORTRANUNDERSCORE")
-#endif()
diff --git a/cime_config/machines/cmake_macros/amdclang_frontier.cmake b/cime_config/machines/cmake_macros/amdclang_frontier.cmake
@@ -1,9 +1,19 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+set(NETCDF_PATH "$ENV{NETCDF_DIR}")
+set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND SLIBS " -L$ENV{PNETCDF_PATH}/lib -lpnetcdf -L$ENV{CRAY_LIBSCI_PREFIX_DIR}/lib -lsci_amd")
 if (NOT MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
 endif()
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
 set(CRAY_LIBSCI_PREFIX_DIR "$ENV{CRAY_LIBSCI_PREFIX_DIR}")
-#if (COMP_NAME STREQUAL gptl)
-#  string(APPEND CPPDEFS " -DFORTRANUNDERSCORE")
-#endif()
diff --git a/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/amdclanggpu_crusher.cmake
@@ -1,3 +1,16 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+set(NETCDF_PATH "$ENV{NETCDF_DIR}")
+set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 if (NOT DEBUG)
   string(APPEND CFLAGS   " -O2")
   string(APPEND CXXFLAGS " -O2")
@@ -16,7 +29,6 @@ string(APPEND SLIBS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/
 if (NOT MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
 endif()
-
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 
 set(USE_HIP "TRUE")

diff --git a/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/amdclanggpu_frontier.cmake
@@ -1,3 +1,16 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+set(NETCDF_PATH "$ENV{NETCDF_DIR}")
+set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 if (NOT DEBUG)
   string(APPEND CFLAGS   " -O2")
   string(APPEND CXXFLAGS " -O2")
@@ -13,7 +26,9 @@ set(MPICXX "hipcc")
 set(SCXX "hipcc")
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
-
+if (NOT MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
+endif()
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 
 set(USE_HIP "TRUE")

diff --git a/cime_config/machines/cmake_macros/crayclang.cmake b/cime_config/machines/cmake_macros/crayclang.cmake
@@ -25,9 +25,16 @@ set(HAS_F2008_CONTIGUOUS "TRUE")
 string(APPEND LDFLAGS " -Wl,--allow-multiple-definition")
 set(SUPPORTS_CXX "TRUE")
 set(CXX_LINKER "FORTRAN")
+
 set(MPICC "cc")
 set(MPICXX "CC")
 set(MPIFC "ftn")
 set(SCC "cc")
 set(SCXX "CC")
 set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
diff --git a/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake b/cime_config/machines/cmake_macros/crayclanggpu_crusher.cmake
@@ -1,3 +1,15 @@
+set(MPICC "cc")
+set(MPICXX "hipcc")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "hipcc")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
 if (compile_threaded)
   string(APPEND FFLAGS   " -fopenmp")
   string(APPEND CFLAGS   " -fopenmp")
@@ -29,12 +41,6 @@ string(APPEND LDFLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefi
 
 set(SUPPORTS_CXX "TRUE")
 set(CXX_LINKER "FORTRAN")
-set(MPICC "cc")
-set(MPICXX "hipcc")
-set(MPIFC "ftn")
-set(SCC "cc")
-set(SCXX "hipcc")
-set(SFC "ftn")
 
 # Switch to O3 for better performance
 # Using O2 to ensure passing tests
@@ -61,6 +67,7 @@ string(APPEND CXX_LIBS " -lstdc++")
 
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
+string(APPEND SLIBS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
 if (NOT MPILIB STREQUAL mpi-serial)
   string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
 endif()

diff --git a/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake b/cime_config/machines/cmake_macros/crayclanggpu_frontier.cmake
@@ -1,3 +1,15 @@
+set(MPICC "cc")
+set(MPICXX "hipcc")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "hipcc")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+
 if (compile_threaded)
   string(APPEND FFLAGS   " -fopenmp")
   string(APPEND CFLAGS   " -fopenmp")
@@ -29,12 +41,6 @@ string(APPEND LDFLAGS " -Wl,--allow-multiple-definition -Wl,--allow-shlib-undefi
 
 set(SUPPORTS_CXX "TRUE")
 set(CXX_LINKER "FORTRAN")
-set(MPICC "cc")
-set(MPICXX "hipcc")
-set(MPIFC "ftn")
-set(SCC "cc")
-set(SCXX "hipcc")
-set(SFC "ftn")
 
 # Switching to O3 for performance benchmarking
 # Will revisit any failing tests
@@ -53,8 +59,6 @@ endif()
 # Disable ipa and zero initialization are for other NaN isues:
 # https://github.com/E3SM-Project/E3SM/pull/5208
 string(APPEND FFLAGS " -hipa0 -hzero -em -ef -hnoacc")
-# Solving a 15.0.0 build issue
-#string(APPEND FFLAGS " -hsystem_alloc")
 
 set(NETCDF_PATH "$ENV{NETCDF_DIR}")
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
@@ -63,8 +67,11 @@ string(APPEND CXX_LIBS " -lstdc++")
 
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa")
-# string(APPEND SLIBS " -L$ENV{ROCM_PATH}/lib -lamdhip64 -L$ENV{OLCF_OPENBLAS_ROOT}/lib -lopenblas $ENV{OLCF_LIBUNWIND_ROOT}/lib/libunwind.a /sw/frontier/spack-envs/base/opt/cray-sles15-zen3/clang-14.0.0-rocm5.2.0/gperftools-2.10-6g5acp4pcilrl62tddbsbxlut67pp7qn/lib/libtcmalloc.a")
 string(APPEND SLIBS " -L$ENV{ROCM_PATH}/lib -lamdhip64")
+if (NOT MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
+endif()
+
 
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 

diff --git a/cime_config/machines/cmake_macros/gnu_crusher.cmake b/cime_config/machines/cmake_macros/gnu_crusher.cmake
@@ -1,3 +1,14 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND FFLAGS " -Wno-implicit-interface")
 
 if (NOT DEBUG)

diff --git a/cime_config/machines/cmake_macros/gnu_frontier.cmake b/cime_config/machines/cmake_macros/gnu_frontier.cmake
@@ -1,3 +1,14 @@
+set(MPICC "cc")
+set(MPICXX "CC")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "CC")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND FFLAGS " -Wno-implicit-interface")
 
 if (NOT DEBUG)

diff --git a/cime_config/machines/cmake_macros/gnugpu_crusher.cmake b/cime_config/machines/cmake_macros/gnugpu_crusher.cmake
@@ -1,24 +1,25 @@
+set(MPICC "cc")
+set(MPICXX "hipcc")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "hipcc")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND FFLAGS " -Wno-implicit-interface")
 
 if (NOT DEBUG)
   string(APPEND CFLAGS   " -O2")
   string(APPEND CXXFLAGS " -O2")
   string(APPEND FFLAGS   " -O2")
 endif()
-if (COMP_NAME STREQUAL gptl)
-  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
-endif()
 set(NETCDF_PATH "$ENV{NETCDF_DIR}")
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
 string(APPEND CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
 
-set(MPICC  "cc")
-set(MPICXX "hipcc")
-set(MPIFC  "ftn")
-set(SCC  "cc")
-set(SCXX "hipcc")
-set(SFC  "ftn")
-
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ")
 if (NOT MPILIB STREQUAL mpi-serial)

diff --git a/cime_config/machines/cmake_macros/gnugpu_frontier.cmake b/cime_config/machines/cmake_macros/gnugpu_frontier.cmake
@@ -1,26 +1,30 @@
+set(MPICC "cc")
+set(MPICXX "hipcc")
+set(MPIFC "ftn")
+set(SCC "cc")
+set(SCXX "hipcc")
+set(SFC "ftn")
+
+string(APPEND CPPDEFS " -DLINUX")
+if (COMP_NAME STREQUAL gptl)
+    string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
 string(APPEND FFLAGS " -Wno-implicit-interface")
 
 if (NOT DEBUG)
   string(APPEND CFLAGS   " -O2")
   string(APPEND CXXFLAGS " -O2")
   string(APPEND FFLAGS   " -O2")
 endif()
-if (COMP_NAME STREQUAL gptl)
-  string(APPEND CPPDEFS " -DHAVE_SLASHPROC")
-endif()
 set(NETCDF_PATH "$ENV{NETCDF_DIR}")
 set(PNETCDF_PATH "$ENV{PNETCDF_DIR}")
 string(APPEND CMAKE_OPTS " -DPIO_ENABLE_TOOLS:BOOL=OFF")
 
-set(MPICC  "cc")
-set(MPICXX "hipcc")
-set(MPIFC  "ftn")
-set(SCC  "cc")
-set(SCXX "hipcc")
-set(SFC  "ftn")
-
 string(APPEND CXXFLAGS " -I$ENV{MPICH_DIR}/include --offload-arch=gfx90a")
 string(APPEND SLIBS    " -Wl,--copy-dt-needed-entries -L/opt/cray/pe/gcc-libs -lgfortran -L$ENV{MPICH_DIR}/lib -lmpi -L$ENV{CRAY_MPICH_ROOTDIR}/gtl/lib -lmpi_gtl_hsa ")
+if (NOT MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -L$ENV{ADIOS2_DIR}/lib64 -ladios2_c_mpi -ladios2_c -ladios2_core_mpi -ladios2_core -ladios2_evpath -ladios2_ffs -ladios2_dill -ladios2_atl -ladios2_enet")
+endif()
 
 string(APPEND KOKKOS_OPTIONS " -DKokkos_ENABLE_HIP=On -DKokkos_ARCH_ZEN3=On -DKokkos_ARCH_VEGA90A=On")
 

diff --git a/cime_config/machines/config_machines.xml b/cime_config/machines/config_machines.xml
@@ -518,7 +518,7 @@
   <machine MACH="spock">
     <DESC>Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL.</DESC>
     <NODENAME_REGEX>.*spock.*</NODENAME_REGEX>
-    <OS>CNL</OS>
+    <OS>Linux</OS>
     <COMPILERS>gnu,cray</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli133</PROJECT>
@@ -599,8 +599,8 @@
   <machine MACH="frontier">
     <DESC>Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 </DESC>
     <NODENAME_REGEX>.*frontier.*</NODENAME_REGEX>
-    <OS>CNL</OS>
-    <COMPILERS>gnu,crayclang,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
+    <OS>Linux</OS>
+    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/cli115/world-shared/frontier</SAVE_TIMING_DIR>
@@ -723,8 +723,8 @@
   <machine MACH="crusher">
     <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
     <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>CNL</OS>
-    <COMPILERS>gnu,crayclang,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
+    <OS>Linux</OS>
+    <COMPILERS>crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>cli115</PROJECT>
     <SAVE_TIMING_DIR>/lustre/orion/cli115/world-shared/crusher</SAVE_TIMING_DIR>
@@ -846,7 +846,7 @@
   <machine MACH="crusher-scream-cpu">
     <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
     <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>CNL</OS>
+    <OS>Linux</OS>
     <COMPILERS>crayclang-scream</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>CLI133_crusher</PROJECT>
@@ -945,7 +945,7 @@
   <machine MACH="crusher-scream-gpu">
     <DESC>Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs</DESC>
     <NODENAME_REGEX>.*crusher.*</NODENAME_REGEX>
-    <OS>CNL</OS>
+    <OS>Linux</OS>
     <COMPILERS>crayclang-scream</COMPILERS>
     <MPILIBS>mpich</MPILIBS>
     <PROJECT>CLI133_crusher</PROJECT>