From ca5b34653701fe09dc685a65b2313744f2bb3810 Mon Sep 17 00:00:00 2001 From: Matthew Thompson Date: Wed, 15 Jan 2025 11:09:14 -0500 Subject: [PATCH] v11: Add Cas for SLES15; Add back IMPI flags on SLES15 --- gcm_setup | 15 ++++++++++++++- geoschemchem_setup | 15 ++++++++++++++- gmichem_setup | 15 ++++++++++++++- stratchem_setup | 15 ++++++++++++++- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/gcm_setup b/gcm_setup index d7e379b1..04a9d636 100755 --- a/gcm_setup +++ b/gcm_setup @@ -420,6 +420,7 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then echo "Enter the ${C1}Processor Type${CN} you wish to run on:" echo " ${C2}mil (Milan)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" echo " " set MODEL = `echo $<` set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` @@ -427,11 +428,20 @@ if ( $SITE == 'NCCS' ) then set MODEL = 'mil' endif - if( $MODEL != 'mil' ) goto ASKPROC + if( $MODEL != 'mil' & \ + $MODEL != 'cas' ) goto ASKPROC if ($MODEL == 'mil') then # We save a couple processes for the kernel set NCPUS_PER_NODE = 126 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 endif else echo "Enter the ${C1}Processor Type${CN} you wish to run on:" @@ -2279,6 +2289,9 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + setenv I_MPI_FABRICS shm:ofi setenv I_MPI_OFI_PROVIDER psm3 EOF diff --git a/geoschemchem_setup b/geoschemchem_setup index 77b5d553..73f23cb9 100755 --- a/geoschemchem_setup +++ b/geoschemchem_setup @@ -420,6 +420,7 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then echo "Enter the ${C1}Processor Type${CN} you wish to run on:" echo " ${C2}mil (Milan)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" echo " " set MODEL = `echo $<` set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` @@ -427,11 +428,20 @@ if ( $SITE == 'NCCS' ) then set MODEL = 'mil' endif - if( $MODEL != 'mil' ) goto ASKPROC + if( $MODEL != 'mil' & \ + $MODEL != 'cas' ) goto ASKPROC if ($MODEL == 'mil') then # We save a couple processes for the kernel set NCPUS_PER_NODE = 126 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 endif else echo "Enter the ${C1}Processor Type${CN} you wish to run on:" @@ -2309,6 +2319,9 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + setenv I_MPI_FABRICS shm:ofi setenv I_MPI_OFI_PROVIDER psm3 EOF diff --git a/gmichem_setup b/gmichem_setup index 3858c3b3..70aabfb0 100755 --- a/gmichem_setup +++ b/gmichem_setup @@ -420,6 +420,7 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then echo "Enter the ${C1}Processor Type${CN} you wish to run on:" echo " ${C2}mil (Milan)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" echo " " set MODEL = `echo $<` set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` @@ -427,11 +428,20 @@ if ( $SITE == 'NCCS' ) then set MODEL = 'mil' endif - if( $MODEL != 'mil' ) goto ASKPROC + if( $MODEL != 'mil' & \ + $MODEL != 'cas' ) goto ASKPROC if ($MODEL == 'mil') then # We save a couple processes for the kernel set NCPUS_PER_NODE = 126 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 endif else echo "Enter the ${C1}Processor Type${CN} you wish to run on:" @@ -2481,6 +2491,9 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + setenv I_MPI_FABRICS shm:ofi setenv I_MPI_OFI_PROVIDER psm3 EOF diff --git a/stratchem_setup b/stratchem_setup index 9d168df1..0b0ea880 100755 --- a/stratchem_setup +++ b/stratchem_setup @@ -420,6 +420,7 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then echo "Enter the ${C1}Processor Type${CN} you wish to run on:" echo " ${C2}mil (Milan)${CN} (default)" + echo " ${C2}cas (Cascade Lake)${CN}" echo " " set MODEL = `echo $<` set MODEL = `echo $MODEL | tr "[:upper:]" "[:lower:]"` @@ -427,11 +428,20 @@ if ( $SITE == 'NCCS' ) then set MODEL = 'mil' endif - if( $MODEL != 'mil' ) goto ASKPROC + if( $MODEL != 'mil' & \ + $MODEL != 'cas' ) goto ASKPROC if ($MODEL == 'mil') then # We save a couple processes for the kernel set NCPUS_PER_NODE = 126 + else if ($MODEL == 'cas') then + # NCCS currently recommends that users do not run with + # 48 cores per node on SCU16 due to OS issues and + # recommends that CPU-intensive works run with 46 or less + # cores. As 45 is a multiple of 3, it's the best value + # that doesn't waste too much + #set NCPUS_PER_NODE = 48 + set NCPUS_PER_NODE = 45 endif else echo "Enter the ${C1}Processor Type${CN} you wish to run on:" @@ -2294,6 +2304,9 @@ if ( $SITE == 'NCCS' ) then if ("$BUILT_ON_SLES15" == "TRUE") then cat >> $HOMDIR/SETENV.commands << EOF +setenv I_MPI_ADJUST_ALLREDUCE 12 +setenv I_MPI_ADJUST_GATHERV 3 + setenv I_MPI_FABRICS shm:ofi setenv I_MPI_OFI_PROVIDER psm3 EOF