Merge pull request #526 from GEOS-ESM/bugfix/mathomp4/fix-coupled-darwin

Fixes for running coupled on macOS
GEOS-ESM · Oct 19, 2023 · d3c0b4b · d3c0b4b
2 parents 826aa0c + 046c9b2
commit d3c0b4b
Show file tree

Hide file tree

Showing 13 changed files with 118 additions and 70 deletions.
diff --git a/gcm_archive.j b/gcm_archive.j
@@ -33,9 +33,9 @@ setenv SITE             @SITE
 setenv GEOSBIN          @GEOSBIN
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
 
 #######################################################################
-#                         Archive Commands           
+#                         Archive Commands
 #######################################################################
 
diff --git a/gcm_forecast.tmpl b/gcm_forecast.tmpl
@@ -32,9 +32,9 @@ setenv GEOSETC          @GEOSETC
 setenv GEOSUTIL         @GEOSSRC
 
 @NATIVE_BUILD source $GEOSBIN/g5_modules
-@NATIVE_BUILD setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
+@NATIVE_BUILD setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
 
-setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+setenv RUN_CMD "@RUN_CMD"
 
 setenv GCMVER `cat $GEOSETC/.AGCM_VERSION`
 echo   VERSION: $GCMVER

diff --git a/gcm_moveplot.j b/gcm_moveplot.j
@@ -34,7 +34,7 @@ setenv GEOSBIN          @GEOSBIN
 setenv GEOSUTIL         @GEOSSRC
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
 
 #######################################################################
 #             Experiment Specific Environment Variables

diff --git a/gcm_plot.tmpl b/gcm_plot.tmpl
@@ -34,7 +34,7 @@ setenv GEOSBIN          @GEOSBIN
 setenv GEOSUTIL         @GEOSSRC
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
 
 #######################################################################
 #             Experiment Specific Environment Variables
@@ -45,7 +45,7 @@ setenv    EXPDIR  @EXPDIR
 setenv    HOMDIR  @HOMDIR
 
 #######################################################################
-#                         Quickplot Commands           
+#                         Quickplot Commands
 #######################################################################
 
 cd $EXPDIR/plot
@@ -57,7 +57,7 @@ else
 endif
 
 #######################################################################
-#                        Move Plots to Web Server   
+#                        Move Plots to Web Server
 #######################################################################
 
  set MOVE =  `grep PLOT_MOVE: $EXPDIR/plot/plot.rc | cut -d'#' -f1 | cut -d':' -f2`

diff --git a/gcm_post.j b/gcm_post.j
@@ -36,13 +36,13 @@ setenv GEOSUTIL         @GEOSSRC
 setenv BATCHNAME       "@POST_N"
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
 
 if( $?SLURM_NTASKS ) then
-      setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+      setenv RUN_CMD "@RUN_CMD"
       set NCPUS = $SLURM_NTASKS
 else if( $?PBS_NODEFILE ) then
-      setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+      setenv RUN_CMD "@RUN_CMD"
       set NCPUS = `cat $PBS_NODEFILE | wc -l`
 else
       set NCPUS = NULL

diff --git a/gcm_quickstat.j b/gcm_quickstat.j
@@ -29,10 +29,10 @@ setenv GEOSBIN          @GEOSBIN
 setenv GEOSUTIL         @GEOSSRC
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib
 
 #######################################################################
-#                         Quickstat Commands           
+#                         Quickstat Commands
 #######################################################################
 
 cd $GEOSUTIL/plots

diff --git a/gcm_regress.j b/gcm_regress.j
@@ -90,9 +90,9 @@ setenv GEOSDIR @GEOSDIR
 setenv GEOSBIN @GEOSBIN
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
 
-setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+setenv RUN_CMD "@RUN_CMD"
 
 #######################################################################
 #             Experiment Specific Environment Variables

diff --git a/gcm_run.j b/gcm_run.j
@@ -33,9 +33,9 @@ setenv GEOSETC          @GEOSETC
 setenv GEOSUTIL         @GEOSSRC
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
 
-setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+setenv RUN_CMD "@RUN_CMD"
 
 setenv GCMVER `cat $GEOSETC/.AGCM_VERSION`
 echo   VERSION: $GCMVER
@@ -325,7 +325,7 @@ setenv EMISSIONS @EMISSIONS
 @MOM5setenv BCTAG `basename $ABCSDIR`
 @MOM6setenv BCTAG `basename $ABCSDIR`
 #this is hard-wired for NAS for now - should make it more general
-@MITsetenv GRIDDIR /nobackupp18/afahad/GEOSMITgcmFiles/GRIDDIR/a${AGCM_IM}x${AGCM_JM}_o${OGCM_IM}x${OGCM_JM} 
+@MITsetenv GRIDDIR /nobackupp18/afahad/GEOSMITgcmFiles/GRIDDIR/a${AGCM_IM}x${AGCM_JM}_o${OGCM_IM}x${OGCM_JM}
 @MITsetenv BCTAG `basename $GRIDDIR`
 @DATAOCEANsetenv BCTAG `basename $BCSDIR`
 
@@ -545,15 +545,15 @@ set tile_rsts = (catch catchcn route lake landice openwater saltwater seaicether
 # ----------------------------------
 set rst_by_face = NO
 if( $GCMEMIP == TRUE ) then
-   if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_rst & -e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then 
+   if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_rst & -e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then
      echo "grid-based internal_rst and internal_face_x_rst should not co-exist"
      echo "please remove all *internal_rst except these tile-based restarts :"
      foreach rst ( $tile_rsts )
         echo ${rst}_internal_rst
      end
      exit
    endif
-   if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then 
+   if(-e $EXPDIR/restarts/$RSTDATE/fvcore_internal_face_1_rst) then
      set rst_by_face = YES
    endif
 else
@@ -565,7 +565,7 @@ else
      end
      exit
    endif
-   if(-e $EXPDIR/fvcore_internal_face_1_rst) then 
+   if(-e $EXPDIR/fvcore_internal_face_1_rst) then
      set rst_by_face = YES
    endif
 endif
@@ -610,7 +610,7 @@ foreach rst ( $dummy )
        if ( $rst =~ *$tile_rst* ) then
          set is_tile_rst = TRUE
          break
-       endif  
+       endif
      end
   endif
   if ($is_tile_rst == FALSE & $rst_by_face == YES) then
@@ -619,7 +619,7 @@ foreach rst ( $dummy )
          set rst = ${part1}_face_${n}_rst
          set rst_file_names = `echo $rst_file_names $rst`
       end
-  else   
+  else
     set rst_file_names = `echo $rst_file_names $rst`
   endif
 end
@@ -1005,7 +1005,7 @@ endif
 
 # Run bundleParser.py
 #---------------------
-python bundleParser.py
+python3 bundleParser.py
 
 # If REPLAY, link necessary forcing files
 # ---------------------------------------
@@ -1039,53 +1039,53 @@ endif
 @MIT # ---------------------------------------------------
 @MIT # For MITgcm restarts - before running GEOSgcm.x
 @MIT # ---------------------------------------------------
-@MIT 
+@MIT
 @MIT # set time interval for segment in seconds
-@MIT 
+@MIT
 @MIT set yearc  = `echo $nymdc | cut -c1-4`
 @MIT set monthc = `echo $nymdc | cut -c5-6`
 @MIT set dayc   = `echo $nymdc | cut -c7-8`
 @MIT set hourc  = `echo $nhmsc | cut -c1-2`
 @MIT set minutec = `echo $nhmsc | cut -c3-4`
 @MIT set secondc = `echo $nhmsc | cut -c5-6`
-@MIT 
+@MIT
 @MIT set yearf  = `echo $nymdf | cut -c1-4`
 @MIT set monthf = `echo $nymdf | cut -c5-6`
 @MIT set dayf   = `echo $nymdf | cut -c7-8`
 @MIT set hourf  = `echo $nhmsf | cut -c1-2`
 @MIT set minutef = `echo $nhmsf | cut -c3-4`
 @MIT set secondf = `echo $nhmsf | cut -c5-6`
-@MIT 
+@MIT
 @MIT set yearf = `echo $nymdf | cut -c1-4`
-@MIT 
+@MIT
 @MIT set time1 = `date -u -d "${yearc}-${monthc}-${dayc}T${hourc}:${minutec}:${secondc}" "+%s"`
 @MIT set time2 = `date -u -d "${yearf}-${monthf}-${dayf}T${hourf}:${minutef}:${secondf}" "+%s"`
-@MIT 
+@MIT
 @MIT      @ mitdt = $time2 - $time1
 @MIT echo "Segment time: $mitdt"
-@MIT 
-@MIT 
+@MIT
+@MIT
 @MIT # Set-up MITgcm run directory
 @MIT if (! -e mitocean_run) mkdir -p mitocean_run
 @MIT cd mitocean_run
-@MIT 
+@MIT
 @MIT # link mit configuration and initialization files
 @MIT ln -sf $EXPDIR/mit_input/* .
 @MIT # link mitgcm restarts if exist
 @MIT /bin/ln -sf $EXPDIR/restarts/pic* .
 @MIT # make an archive folder for mitgcm run
 @MIT mkdir $EXPDIR/mit_output
-@MIT 
+@MIT
 @MIT # Calculate segment time steps
 @MIT set mit_nTimeSteps = `cat ${SCRDIR}/AGCM.rc | grep OGCM_RUN_DT: | cut -d: -f2 | tr -s " " | cut -d" " -f2`
 @MIT @ mit_nTimeSteps = ${mitdt} / $mit_nTimeSteps
-@MIT 
+@MIT
 @MIT #change namelist variables in data - nTimeSteps, chkptFreq and monitorFreq
 @MIT sed -i "s/nTimeSteps.*/nTimeSteps       = ${mit_nTimeSteps},/" data
 @MIT sed -i "s/chkptFreq.*/chkptFreq        = ${mitdt}.0,/" data
 @MIT sed -i "s/pChkptFreq.*/pChkptFreq        = ${mitdt}.0,/" data
 @MIT # get nIter0
-@MIT 
+@MIT
 @MIT if (! -e ${EXPDIR}/restarts/MITgcm_restart_dates.txt ) then
 @MIT   set nIter0 = `grep nIter0 data | tr -s " " | cut -d"=" -f2 | cut -d"," -f1 | awk '{$1=$1;print}'`
 @MIT else
@@ -1100,7 +1100,7 @@ endif
 @MIT     sed -i "s/nIter0.*/ nIter0           = ${nIter0},/" data
 @MIT   endif
 @MIT endif
-@MIT 
+@MIT
 @MIT cd ..
 @MIT # ---------------------------------------------------
 @MIT # End MITgcm restarts - before running GEOSgcm.x
@@ -1137,29 +1137,29 @@ echo GEOSgcm Run Status: $rc
 @MIT # ---------------------------------------------------
 @MIT # For MITgcm restarts - after running GEOSgcm.x
 @MIT # ---------------------------------------------------
-@MIT 
+@MIT
 @MIT set STEADY_STATE_OCEAN=`grep STEADY_STATE_OCEAN AGCM.rc | cut -d':' -f2 | tr -d " "`
-@MIT 
+@MIT
 @MIT # update ocean only if activated. Otherwize use the same pickups (passive ocean).
 @MIT if ( ${STEADY_STATE_OCEAN} != 0 ) then
-@MIT 
+@MIT
 @MIT   if ( ${rc} == 0 ) then
-@MIT 
+@MIT
 @MIT     # Update nIter0 for next segment
 @MIT     set znIter00 = `echo $nIter0 | awk '{printf("%010d",$1)}'`
 @MIT     @ nIter0 = $nIter0 + $mit_nTimeSteps
 @MIT     set znIter0 = `echo $nIter0 | awk '{printf("%010d",$1)}'`
-@MIT 
+@MIT
 @MIT     # to update MITgcm restart list file
 @MIT     sed -i "/${nIter0}/d" ${EXPDIR}/restarts/MITgcm_restart_dates.txt
 @MIT     echo "Date_GEOS5 $nymdf $nhmsf NITER0_MITgcm ${nIter0}" >> ${EXPDIR}/restarts/MITgcm_restart_dates.txt
-@MIT 
+@MIT
 @MIT     /bin/mv $SCRDIR/mitocean_run/STDOUT.0000 $EXPDIR/mit_output/STDOUT.${znIter00}
-@MIT 
+@MIT
 @MIT   endif
-@MIT 
+@MIT
 @MIT   cd $SCRDIR/mitocean_run
-@MIT 
+@MIT
 @MIT   # Check existance of roling pickups
 @MIT   set nonomatch rp =  ( pickup*ckptA* )
 @MIT   echo $rp
@@ -1173,7 +1173,7 @@ echo GEOSgcm Run Status: $rc
 @MIT       /bin/mv ${fname} $EXPDIR/restarts/${bname}.${timeStepNumber}.${aname}
 @MIT     end
 @MIT   endif
-@MIT 
+@MIT
 @MIT   # Check existance of permanent pickups
 @MIT   set nonomatch pp =  ( pickup* )
 @MIT   echo $pp
@@ -1183,39 +1183,39 @@ echo GEOSgcm Run Status: $rc
 @MIT       if ( ! -e $EXPDIR/restarts/${fname} ) /bin/mv ${fname} $EXPDIR/restarts/${fname}
 @MIT     end
 @MIT   endif
-@MIT 
+@MIT
 @MIT   /bin/mv T.* $EXPDIR/mit_output/
 @MIT   /bin/mv S.* $EXPDIR/mit_output/
 @MIT   /bin/mv U.* $EXPDIR/mit_output/
 @MIT   /bin/mv V.* $EXPDIR/mit_output/
 @MIT   /bin/mv W.* $EXPDIR/mit_output/
 @MIT   /bin/mv PH* $EXPDIR/mit_output/
 @MIT   /bin/mv Eta.* $EXPDIR/mit_output/
-@MIT 
+@MIT
 @MIT   /bin/mv AREA.* $EXPDIR/mit_output/
 @MIT   /bin/mv HEFF.* $EXPDIR/mit_output/
 @MIT   /bin/mv HSNOW.* $EXPDIR/mit_output/
 @MIT   /bin/mv UICE.* $EXPDIR/mit_output/
 @MIT   /bin/mv VICE.* $EXPDIR/mit_output/
-@MIT 
+@MIT
 @MIT   #copy mit output to mit_output
 @MIT   foreach i (`grep -i filename data.diagnostics  | grep "^ " | cut -d"=" -f2 | cut -d"'" -f2 | awk '{$1=$1;print}'`)
 @MIT    /bin/mv ${i}* $EXPDIR/mit_output/
 @MIT   end
-@MIT 
+@MIT
 @MIT   foreach i (`grep -i stat_fName data.diagnostics | grep "^ " | cut -d"=" -f2 | cut -d"'" -f2 | awk '{$1=$1;print}'`)
 @MIT    /bin/mv ${i}* $EXPDIR/mit_output/
 @MIT   end
-@MIT 
+@MIT
 @MIT   cd $SCRDIR
-@MIT 
+@MIT
 @MIT endif
-@MIT 
+@MIT
 @MIT # ---------------------------------------------------
 @MIT # End MITgcm restarts - after running GEOSgcm.x
 @MIT # ---------------------------------------------------
 
- 
+
 #######################################################################
 #   Rename Final Checkpoints => Restarts for Next Segment and Archive
 #        Note: cap_restart contains the current NYMD and NHMS
@@ -1314,12 +1314,12 @@ end
 @CICE6 set dsets="iceh"
 @CICE6 foreach dset ( $dsets )
 @CICE6  set num = `/bin/ls -1 $dset.*.nc | wc -l`
-@CICE6  if($num != 0) then    
+@CICE6  if($num != 0) then
 @CICE6     if(! -e $EXPDIR/CICE_Output) mkdir -p $EXPDIR/CICE_Output
 @CICE6     /bin/mv $SCRDIR/$dset.*.nc $EXPDIR/CICE_Output/
 @CICE6  endif
-@CICE6 end     
-@CICE6     
+@CICE6 end
+@CICE6
 #######################################################################
 #                 Run Post-Processing and Forecasts
 #######################################################################

diff --git a/gcm_run_benchmark.j b/gcm_run_benchmark.j
@@ -33,9 +33,9 @@ setenv GEOSETC          @GEOSETC
 setenv GEOSUTIL         @GEOSSRC
 
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
 
-setenv RUN_CMD "$GEOSBIN/esma_mpirun -np "
+setenv RUN_CMD "@RUN_CMD"
 
 setenv GCMVER `cat $GEOSETC/.AGCM_VERSION`
 echo   VERSION: $GCMVER
@@ -951,8 +951,8 @@ setenv GEOSBIN   $GEOSBIN
 setenv GEOSETC   $GEOSETC
 setenv GEOSUTIL  $GEOSUTIL
 source $GEOSBIN/g5_modules
-setenv LD_LIBRARY_PATH ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
-echo $LD_LIBRARY_PATH
+setenv @LD_LIBRARY_PATH_CMD ${LD_LIBRARY_PATH}:${BASEDIR}/${ARCH}/lib:${GEOSDIR}/lib
+echo $@LD_LIBRARY_PATH_CMD
 cd $statsdir
 $RUN_CMD 1 $GEOSUTIL/bin/stats.x -fcst $fcst_files -ana $ana_files -cli $clim_files -rc $GEOSUTIL/post/stats.rc \
                                  -levs $levs_clim -tag $EXPID -nfreq 060000 -fhour $fHOURS