-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbuild.csh
executable file
·976 lines (853 loc) · 29.2 KB
/
build.csh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
#!/bin/tcsh -f
#------------------------------------------------------------------------
# name: parallel_build.csh
# purpose: This subroutine builds using install
# for compilation parallelization.
#
#
# !REVISION HISTORY
# 15Mar2007 Stassi Modified version of g5das_parallel_build.csh
# 22Mar2007 Kokron Add support for discover
# 28Mar2007 Stassi Attempt to find default group for job submittal
# 13Apr2007 Stassi Change order in which LOG/info names get iterated
# 04May2007 Stassi Hard-code NCPUS=4 for discover
# 10Aug2007 TO/JS Added distclean option; reordered script
# 15Apr2008 Stassi Added code to cd to src directory, if not already
# there. Rename ESMASRC variable to ESMADIR
# 16Apr2008 Stassi Use only half of CPUs on palm; mpi needs more memory
# 30Mar2008 Stassi Add runtime options: -h, -hdf, -q queue
# 13Apr2009 Stassi Updated for pleiades
# 05Aug2009 Stassi Added -debug and -tmpdir flags
# 19Aug2010 Stassi Added -walltime flag
# 01Apr2011 Stassi Added clean and realclean options
# 04Nov2014 MAT Moved to sbatch on discover, added Haswell
# as an option, removed some older batch systems
# 07Jul2016 MAT Added Broadwell at NAS. Removed Westmere. Made
# Broadwell default at NAS.
# 10Oct2017 MAT Added Skylake at NAS. Added option to pass in
# account
# 08Jul2019 MAT Changes for git-based GEOSadas
#------------------------------------------------------------------------
# NOTE: This is now called by a stub routine. For sanity's
# sake, we fake its original name here so that the stub
# will print the right name in usage output
set name = 'parallel_build.csh'
set scriptname = $name
set BUILD_LOG_DIR = BUILD_LOG_DIR
setenv ARCH `uname -s`
#--set time = (1000 "%Uu %Ss %E %X+%Dk %Mk %I+%Oio %Fpf+%Ww")
set time = ( 1000 "%Uu %Ss %E" )
set NCPUs_min = 6
#=====================
# determine the site
#=====================
set node = `uname -n`
if ( ($node == dirac) \
|| ($node =~ borg*) \
|| ($node =~ warp*) \
|| ($node =~ discover*)) then
setenv SITE NCCS
else if (($node =~ pfe*) \
|| ($node =~ tfe*) \
|| ($node =~ r[0-9]*i[0-9]*n[0-9]*) \
|| ($node =~ r[0-9]*c[0-9]*t[0-9]*n[0-9]*)) then
setenv SITE NAS
else
setenv SITE UNKNOWN
set NCPUs_min = 1
endif
# set defaults
#-------------
if (! $?ESMADIR) setenv ESMADIR ""
if (! $?docmake) setenv docmake 1
if (! $?usegnu) setenv usegnu 0
if (! $?notar) setenv notar 0
if (! $?ddb) setenv ddb 0
if (! $?debug) setenv debug 0
if (! $?aggressive) setenv aggressive 0
if (! $?verbose) setenv verbose ""
if (! $?interactive) setenv interactive 0
if (! $?do_wait) setenv do_wait 0
if (! $?proc) setenv proc ""
if (! $?prompt) setenv prompt 1
if (! $?queue) setenv queue ""
if (! $?partition) setenv partition ""
if (! $?account) setenv account ""
if (! $?tmpdir) setenv tmpdir ""
if (! $?walltime) setenv walltime ""
if (! $?slurm_constraint) setenv slurm_constraint ""
if (! $?cmake_build_type) setenv cmake_build_type "Release"
if (! $?EXTRA_CMAKE_FLAGS) setenv EXTRA_CMAKE_FLAGS ""
if (! $?FORTRAN_COMPILER) setenv FORTRAN_COMPILER ""
if (! $?BUILDDIR_PASSED) setenv BUILDDIR_PASSED "NO"
if (! $?INSTALLDIR_PASSED) setenv INSTALLDIR_PASSED "NO"
# Detect if on compute node already
# ---------------------------------
setenv oncompnode 0
if ($?PBS_JOBID || $?SLURM_JOBID) then
set oncompnode = 1
endif
# check for runtime parameters
#-----------------------------
@ n = 0
while ($#argv)
# usage information
#------------------
if ("$1" == "-help") goto usage
if ("$1" == "-h" ) goto usage
# developer's debug
#------------------
if ("$1" == "-ddb") set ddb = 1
# compile with debug
#-------------------
if (("$1" == "-debug") || ("$1" == "-db")) then
setenv cmake_build_type "Debug"
setenv debug 1
endif
# compile with aggressive
#------------------------
if ("$1" == "-aggressive") then
setenv cmake_build_type "Aggressive"
setenv aggressive 1
endif
# specify node type
#------------------
if ("$1" == "-mil") set nodeTYPE = "Milan"
if ("$1" == "-rom") set nodeTYPE = "Rome"
if ("$1" == "-cas") set nodeTYPE = "CascadeLake"
if ("$1" == "-sky") set nodeTYPE = "Skylake"
if ("$1" == "-bro") set nodeTYPE = "Broadwell"
if ("$1" == "-has") set nodeTYPE = "Haswell"
#if ("$1" == "-any") set nodeTYPE = "Any node"
# reset Fortran TMPDIR
#---------------------
if ("$1" == "-tmpdir") then
shift; if (! $#argv) goto usage
setenv tmpdir $1
endif
# set ESMADIR
#------------
if ("$1" == "-esmadir") then
shift; if (! $#argv) goto usage
setenv ESMADIR $1
endif
# set BUILDDIR
#-------------
if ("$1" == "-builddir") then
shift; if (! $#argv) goto usage
setenv BUILDDIR $1
setenv BUILDDIR_PASSED "YES"
endif
# set INSTALLDIR
#---------------
if ("$1" == "-installdir") then
shift; if (! $#argv) goto usage
setenv INSTALLDIR $1
setenv INSTALLDIR_PASSED "YES"
endif
# set GMI_MECHANISM
#------------------
if ("$1" == "-gmi_mechanism") then
shift; if (! $#argv) goto usage
setenv GMI_MECHANISM $1
endif
# run job interactively
#----------------------
if ("$1" == "-i") set interactive = 1
# set verbose flag
#-----------------
if ("$1" == "-verbose") set verbose = "VERBOSE=1"
if ("$1" == "-v") set verbose = "VERBOSE=1"
# run job interactively
#----------------------
if ("$1" == "-wait") set do_wait = 1
# submit batch job to alternative queue/qos
#------------------------------------------
if ("$1" == "-q") then
shift; if (! $#argv) goto usage
if ($SITE == NCCS) then
setenv queue "--qos=$1"
else
setenv queue "-q $1"
endif
endif
# submit batch job to specified partition
#----------------------------------------
if ("$1" == "-partition") then
shift; if (! $#argv) goto usage
setenv partition "--partition=$1"
endif
# submit batch job to specified account
#--------------------------------------
if ("$1" == "-account") then
shift; if (! $#argv) goto usage
setenv account "$1"
endif
# set batch walltime
#-------------------
if ("$1" == "-walltime") then
shift; if (! $#argv) goto usage
setenv walltime $1
set hr = `echo $walltime | cut -d: -f1`
set ss = `echo $walltime | cut -d: -f3`
if (("$hr" == "$walltime") || ("$ss" == "")) then
echo ""
echo walltime must be in hr:mm:ss format
goto usage
endif
endif
# set noprompt option
#--------------------
if (("$1" == "-np") || ("$1" == "-noprompt")) then
setenv prompt 0
endif
# set nocmake option
#--------------------
if ("$1" == "-nocmake") then
setenv docmake 0
endif
# set gnu option
#---------------
if ("$1" == "-gnu") then
setenv usegnu 1
endif
# set no tar option
#------------------
if ("$1" == "-no-tar") then
setenv notar 1
endif
# If a user passes in '-- ' then everything after that is passed
# into EXTRA_CMAKE_FLAGS and we are done parsing arguments
#--------------------------------------------------------------
if ("$1" == "--") then
shift
setenv EXTRA_CMAKE_FLAGS "$*"
set argv = ()
break
endif
shift
end
# Check if the ESMA_NOTAR environment variable is set
# If so, then set notar to 1
# --------------------------------------------------
if ($?ESMA_NOTAR) then
echo "ESMA_NOTAR is set, so not creating tar file"
setenv notar 1
endif
# Only allow one of debug and aggressive
# --------------------------------------
if ( ($aggressive) && ($debug) ) then
echo "ERROR. Only one of -debug and -aggressive is allowed"
exit 1
endif
# default nodeTYPE
#-----------------
if (! $?nodeTYPE) then
if ($SITE == NCCS) set nodeTYPE = "Milan"
if ($SITE == NAS) set nodeTYPE = "Rome"
endif
# at NCCS
#--------
if ($SITE == NCCS) then
set nT = `echo $nodeTYPE| tr "[A-Z]" "[a-z]" | cut -c1-3 `
#if ( ($nT != cas) && ($nT != mil) && ($nT != any) ) then
if ( ($nT != cas) && ($nT != mil) ) then
echo "ERROR. Unknown node type at NCCS: $nodeTYPE"
exit 1
endif
# For the any node, set the default to 40 cores as
# this is the least number of cores you will get
#if ($nT == any) @ NCPUS_DFLT = 48
if ($nT == cas) @ NCPUS_DFLT = 48
if ($nT == mil) @ NCPUS_DFLT = 128
#if ($nT == any) set proc = 'any'
if ($nT == cas) set proc = 'cas'
if ($nT == mil) set proc = 'mil'
# If we are using GNU at NCCS, we can *only* use the cas or mil processors
# as OpenMPI is only built for Infiniband. Keeping this here in case
# more MPI issues are in future
if ($usegnu) then
set slurm_constraint = "--constraint=$proc"
#else if ($nT == any) then
#set slurm_constraint = "--constraint=mil|cas"
else
set slurm_constraint = "--constraint=$proc"
endif
if ("$queue" == "") then
set queue = '--qos=debug'
endif
if ("$partition" == "") then
set partition = '--partition=compute'
endif
endif
# at NAS
#-------
if ( $SITE == NAS ) then
set nT = `echo $nodeTYPE | cut -c1-3 | tr "[A-Z]" "[a-z]"`
if (($nT != has) && ($nT != bro) && ($nT != sky) && ($nT != cas) && ($nT != rom) && ($nT != mil)) then
echo "ERROR. Unknown node type at NAS: $nodeTYPE"
exit 2
endif
if ($nT == mil) set nT = 'mil_ait'
if ($nT == rom) set nT = 'rom_ait'
if ($nT == sky) set nT = 'sky_ele'
if ($nT == cas) set nT = 'cas_ait'
set proc = ":model=$nT"
if ($nT == has) @ NCPUS_DFLT = 24
if ($nT == bro) @ NCPUS_DFLT = 28
if ($nT == sky_ele) @ NCPUS_DFLT = 40
if ($nT == cas_ait) @ NCPUS_DFLT = 40
if ($nT == rom_ait) @ NCPUS_DFLT = 128
if ($nT == mil_ait) @ NCPUS_DFLT = 128
# TMPDIR needs to be reset
#-------------------------
if ($tmpdir == '') then
set tmpdirDFLT = "/nobackup/$USER/scratch/"
if ($prompt) then
echo ""
echo -n "Define TMPDIR location "
echo "where scratch files can be written during build."
echo -n "TMPDIR [$tmpdirDFLT] "
setenv tmpdir $<
if ("$tmpdir" == "") setenv tmpdir $tmpdirDFLT
else
setenv tmpdir $tmpdirDFLT
endif
endif
echo "TMPDIR: $tmpdir"
endif
if ($SITE == UNKNOWN) then
echo ""
if ($ARCH == Darwin) then
@ NCPUS_DFLT = `sysctl -a | grep machdep.cpu.core_count | awk '{print $2}'`
else
@ NCPUS_DFLT = `cat /proc/cpuinfo | grep processor | wc -l`
endif
echo "Unknown site. Detected $NCPUS_DFLT cores and setting interactive build"
set interactive = 1
endif
# set Pbuild_build_directory
# --------------------------
if (! $?Pbuild_build_directory) then
if ($?BUILDDIR) then
setenv Pbuild_build_directory $ESMADIR/$BUILDDIR
else if ($debug) then
setenv Pbuild_build_directory $ESMADIR/build-Debug
else if ($aggressive) then
setenv Pbuild_build_directory $ESMADIR/build-Aggressive
else
setenv Pbuild_build_directory $ESMADIR/build
endif
endif
# set Pbuild_install_directory
# ----------------------------
if (! $?Pbuild_install_directory) then
if ($?INSTALLDIR) then
setenv Pbuild_install_directory $ESMADIR/$INSTALLDIR
else if ($debug) then
setenv Pbuild_install_directory $ESMADIR/install-Debug
else if ($aggressive) then
setenv Pbuild_install_directory $ESMADIR/install-Aggressive
else
setenv Pbuild_install_directory $ESMADIR/install
endif
endif
# If we are at NCCS, because of the dual OSs, we decorate the build and
# install directory with the OS name. If we submit to Milan, we will add
# -SLES15, otherwise -SLES12 to the build and install directories. But,
# we only do this if the user has not specified a build directory or
# install directory
# ---------------------------------------------------------------------
if ($SITE == NCCS) then
# We now have to handle this in two ways. One if we are on a compute node and one if we aren't.
# This is because of how this script works where it sort of submits itself to the batch system
# and many of the variables known by the script before submission are lost after submission.
# So if we are on a compute node, we detect the OS version directly, but if we are just submitting on a
# head node, we instead have to just use the processor type passed in. We'll use oncompnode to detect
# which case we are in.
if ($oncompnode) then
set OS_VERSION=`grep VERSION_ID /etc/os-release | cut -d= -f2 | cut -d. -f1 | sed 's/"//g'`
else
if ($nT == mil) then
set OS_VERSION = 15
else
set OS_VERSION = 12
endif
endif
# We also check if we already appended SLES
if (! $?BUILDDIR && "$BUILDDIR_PASSED" == "NO") then
if ($Pbuild_build_directory !~ "*-SLES${OS_VERSION}") then
setenv Pbuild_build_directory ${Pbuild_build_directory}-SLES${OS_VERSION}
endif
endif
if (! $?INSTALLDIR && "$INSTALLDIR_PASSED" == "NO") then
if ($Pbuild_install_directory !~ "*-SLES${OS_VERSION}") then
setenv Pbuild_install_directory ${Pbuild_install_directory}-SLES${OS_VERSION}
endif
endif
endif
if ("$FORTRAN_COMPILER" == "") then
if ($usegnu) then
setenv FORTRAN_COMPILER 'gfortran'
else
setenv FORTRAN_COMPILER 'ifort'
endif
endif
if (! $?INSTALL_SOURCE_TARFILE) then
if ($notar) then
setenv INSTALL_SOURCE_TARFILE "OFF"
else
setenv INSTALL_SOURCE_TARFILE "ON"
endif
endif
if (! $?GMI_MECHANISM_FLAG) then
if ($?GMI_MECHANISM) then
setenv GMI_MECHANISM_FLAG "-DGMI_MECHANISM=$GMI_MECHANISM"
else
setenv GMI_MECHANISM_FLAG ""
endif
endif
# developer's debug
#------------------
if ($ddb) then
echo "ESMADIR = $ESMADIR"
echo "debug = $debug"
echo "aggressive = $aggressive"
echo "verbose = $verbose"
if ($?nodeTYPE) then
echo "nodeTYPE = $nodeTYPE"
endif
if ($?GMI_MECHANISM) then
echo "GMI_MECHANISM = $GMI_MECHANISM"
endif
echo "tmpdir = $tmpdir"
echo "proc = $proc"
echo "interactive = $interactive"
echo "do_wait = $do_wait"
echo "queue = $queue"
if ($SITE == NCCS) then
echo "partition = $partition"
echo "slurm_constraint = $slurm_constraint"
endif
echo "account = $account"
echo "walltime = $walltime"
echo "prompt = $prompt"
echo "nocmake = $docmake"
echo "notar = $notar"
echo "NCPUS_DFLT = $NCPUS_DFLT"
echo "CMAKE_BUILD_TYPE = $cmake_build_type"
echo "EXTRA_CMAKE_FLAGS = $EXTRA_CMAKE_FLAGS"
echo "Build directory = $Pbuild_build_directory"
echo "Install directory = $Pbuild_install_directory"
echo "usegnu = $usegnu"
echo "FORTRAN_COMPILER = $FORTRAN_COMPILER"
echo "INSTALL_SOURCE_TARFILE = $INSTALL_SOURCE_TARFILE"
echo "GMI_MECHANISM_FLAG = $GMI_MECHANISM_FLAG"
echo "BUILDDIR_PASSED = $BUILDDIR_PASSED"
echo "INSTALLDIR_PASSED = $INSTALLDIR_PASSED"
exit
endif
# if user defines TMPDIR ...
#---------------------------
if ("$tmpdir" != "") then
# ... mkdir if it does not exist
#-------------------------------
if (! -d $tmpdir) then
echo "Making TMPDIR directory: $tmpdir"
mkdir -p $tmpdir
if ($status) then
echo ">> Error << mkdir $tmpdir "
exit 3
endif
endif
# ... check that it is writeable
#-------------------------------
if (! -w $tmpdir) then
echo ">> Error << TMPDIR is not writeable: $tmpdir"
exit 4
endif
echo ""
endif
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# JOB SUBMISSION
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
#========
# intro
#========
echo ""
echo " ================"
echo " PARALLEL BUILD "
echo " ================"
echo ""
setenv Pbuild_source_directory $ESMADIR
# Make the BUILD directory
# ------------------------
if (! -d $Pbuild_build_directory) then
echo "Making build directory: $Pbuild_build_directory"
mkdir -p $Pbuild_build_directory
if ($status) then
echo ">> Error << mkdir $Pbuild_build_directory "
exit 5
endif
endif
set jobname = "parallel_build"
#===========================
# query for number of CPUs
#===========================
if ( ($SITE == NCCS) || ($SITE == NAS) ) then
# use set number of CPUs on NCCS or NAS
#--------------------------------------
@ ncpus_val = $NCPUS_DFLT
@ numjobs_val = $ncpus_val / 3 # save some CPUs for memory
# use only even number of CPUs
#-----------------------------
@ check = ( $numjobs_val % 2 )
if ($check == 1) then
@ numjobs_val --
echo "Rounding down to an even $numjobs_val CPUs."
endif
# Are we on a compute node?
# -------------------------
if ( (! $oncompnode) && $interactive) then
# If we aren't don't use all the cores
if ($numjobs_val > 6) @ numjobs_val = 6
else
# Just use 10 CPUs at most. GEOS doesn't support more
# parallelism in the build
if ($numjobs_val > 10) @ numjobs_val = 10
endif
echo ""
echo -n "The build will proceed with $numjobs_val parallel processes on $ncpus_val CPUs"
if ( (! $oncompnode) && $interactive) then
echo " to not monopolize head node."
else
echo "."
endif
else
# how many?
#----------
echo ""
echo -n "Parallel build using how many CPUs ($NCPUs_min minimum)? "
echo -n "[$NCPUS_DFLT] "
set ncpus_val = $<
if ("$ncpus_val" == "") set ncpus_val = $NCPUS_DFLT
echo ""
# check for minimum number of CPUs
#---------------------------------
if ($ncpus_val < $NCPUs_min) then
@ ncpus_val = $NCPUs_min
echo "Defaulting to minimum $ncpus_val CPUs."
endif
# use only even number of CPUs
#-----------------------------
@ check = ( $ncpus_val / 2 ) * 2
if ($check != $ncpus_val) then
@ ncpus_val ++
echo "Rounding up to an even $ncpus_val CPUs."
endif
# save some CPUs for memory
#--------------------------
@ numjobs_val = $ncpus_val - 2
echo -n "The build will proceed with $numjobs_val parallel processes"
echo " on $ncpus_val CPUs."
endif
setenv ncpus $ncpus_val
setenv numjobs $numjobs_val
#=================================================
# check for LOG and info files from previous build
#=================================================
setenv bldlogdir $Pbuild_build_directory/$BUILD_LOG_DIR
setenv cmakelog $bldlogdir/CLOG
setenv buildlog $bldlogdir/LOG
setenv buildinfo $bldlogdir/info
if (! $?cleanFLAG ) setenv cleanFLAG "notset"
if ("$cleanFLAG" == "notset") then
ls $cmakelog $buildlog $buildinfo >& /dev/null
if ($status == 0) then
if ($prompt) then
echo ''
echo 'Previous build detected - Do you want to clean?'
echo '(c) clean: Removes build and install directories, and rebuilds'
echo '(n) no clean'
echo ''
echo " Note: if you have changed MAPL, we recommend doing a clean for safety's sake"
echo ''
echo -n 'Select (c,n) <<c>> '
set do_clean = $<
if ("$do_clean" != "n") then
set do_clean = "c"
endif
else
set do_clean = "c"
endif
if ("$do_clean" == "c") then
setenv cleanFLAG "clean"
echo "Removing build and install directories and re-running CMake before rebuild"
else
setenv cleanFLAG "noclean"
echo "No clean before rebuild"
endif
endif
endif
#==============
# info for user
#==============
echo ""
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
echo "LOG and info will be written to the $bldlogdir directory."
echo "Do the following to check the status/success of the build:"
echo ""
echo " cd $bldlogdir"
echo ""
echo "Note: Check the latest version of LOG for the most recent build info."
echo ""
echo " The installation will be in $Pbuild_install_directory when complete."
echo ""
# build if interactive
#---------------------
if ($?PBS_JOBID || $?SLURM_JOBID) then
goto build
endif
#=============
# submit job
#=============
set groupflag = ""
if ("$account" != "") then
if ($SITE == NAS) then
set groupflag = "-W group_list=$account"
else if ($SITE == NCCS) then
set groupflag = "--account=$account"
endif
else if (-e `which getsponsor` && (! $interactive)) then
set group = ""
set sponsorinfo = `getsponsor`
while ("$group" == "")
@ n = 1
while ($n < $#sponsorinfo)
if ($sponsorinfo[$n] =~ g[0..9]*) then
set group = $sponsorinfo[$n]
#--break # uncomment this line to make 1st entry the default
endif
@ n++
end
getsponsor
if ("$group" != "") then
echo -n "select group: [$group] "
else
echo -n "select group: "
endif
if ($prompt) then
set reponse = $<
if ("$reponse" != "") set group = $reponse
endif
end
set groupflag = "--account=$group"
endif
set waitflag = ""
if ($do_wait) then
if ($SITE == NAS) then
set waitflag = "-W block=true"
else if ($SITE == NCCS) then
set waitflag = "--wait"
endif
endif
if ($interactive) then
goto build
else if ( $SITE == NAS ) then
if ("$walltime" == "") setenv walltime "1:30:00"
set echo
qsub $groupflag $queue \
-N $jobname \
-l select=1:ncpus=${ncpus}:mpiprocs=${numjobs}$proc \
-l walltime=$walltime \
-S /bin/csh \
-V -j oe -k oed \
$waitflag \
$0
unset echo
if ("$waitflag" == "") then
sleep 1
qstat -a | grep $USER
endif
else if ( $SITE == NCCS ) then
if ("$walltime" == "") setenv walltime "1:00:00"
set echo
# NOTE: The weird long export line below is needed at NCCS because of the
# two OSs. For some reason, if you submit a Milan job from a SLES12
# headnode, it was seeing SLES12 module paths. We believe this is
# because SLURM by default exports all the environment
sbatch $groupflag $partition $queue \
$slurm_constraint \
--job-name=$jobname \
--output=$jobname.o%j \
--nodes=1 \
--ntasks=${numjobs} \
--time=$walltime \
--export ESMADIR=${ESMADIR},cmake_build_type=${cmake_build_type},EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS},FORTRAN_COMPILER=${FORTRAN_COMPILER},INSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE},verbose=${verbose},GMI_MECHANISM_FLAG=${GMI_MECHANISM_FLAG},Pbuild_build_directory=${Pbuild_build_directory},Pbuild_install_directory=${Pbuild_install_directory},usegnu=${usegnu},notar=${notar},tmpdir=${tmpdir},docmake=${docmake},debug=${debug},aggressive=${aggressive},BUILDDIR_PASSED=${BUILDDIR_PASSED},INSTALLDIR_PASSED=${INSTALLDIR_PASSED},queue=${queue},partition=${partition},cleanFLAG=${cleanFLAG} \
$waitflag \
$0
unset echo
if ("$waitflag" == "") then
sleep 1
# Add a longer format for the job name for scripting purposes
squeue -a -o "%.10i %.12P %.10q %.30j %.8u %.8T %.10M %.9l %.6D %.6C %R" -u $USER
endif
else
echo $scriptname": batch procedures are not yet defined for node=$node at site=$SITE"
endif
exit
build:
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# BUILD SYSTEM
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if ( $cleanFLAG == "clean" ) then
echo "Removing build and install directories"
rm -rf $Pbuild_build_directory
rm -rf $Pbuild_install_directory
mkdir -p $Pbuild_build_directory
else
echo "Using existing build and install directories"
endif
chdir $Pbuild_build_directory
setenv ARCH `uname -s`
#=================================================
# create $BUILD_LOG_DIR, plus LOG and info files
#=================================================
# create build log directory
#---------------------------
if (! -d $bldlogdir) mkdir -p $bldlogdir
# increment LOG and info files if previous versions exist
#--------------------------------------------------------
ls $cmakelog $buildlog $buildinfo >& /dev/null
if ($status == 0) then
@ next = 0
foreach file ( $cmakelog $buildlog $buildinfo )
@ num = 1
while (-e ${file}.$num)
@ num ++
end
if ($num > $next) @ next = $num
end
set buildinfo = $bldlogdir/info.$next
set buildlog = $bldlogdir/LOG.$next
set cmakelog = $bldlogdir/CLOG.$next
endif
# alias definitions
#------------------
alias echo2 "echo \!* |& tee $buildinfo $buildlog"
alias echo1 "echo \!* |& tee -a $buildinfo"
alias date1 "date |& tee -a $buildinfo"
# initialize files
#-----------------
echo "Writing LOG and info files to directory: $bldlogdir:t"
echo2 ""
#================
# set environment
#================
if ( -d ${ESMADIR}/@env ) then
source $ESMADIR/@env/g5_modules
else if ( -d ${ESMADIR}/env@ ) then
source $ESMADIR/env@/g5_modules
else if ( -d ${ESMADIR}/env ) then
source $ESMADIR/env/g5_modules
endif
# write environment info to build log
#------------------------------------
if ("$tmpdir" != "") setenv TMPDIR $tmpdir
echo1 "======================================"
echo1 `uname -a`
echo1 "ESMADIR: $ESMADIR"
echo1 "BASEDIR: $BASEDIR"
echo1 "SITE: $SITE"
if ($?TMPDIR) then
echo1 "TMPDIR = $TMPDIR"
endif
echo1 "cmake_build_type: $cmake_build_type"
if ("$verbose" != "") then
echo1 "verbose: $verbose"
endif
if ("$queue" != "") then
echo1 "queue: $queue"
endif
if ("$partition" != "") then
echo1 "partition: $partition"
endif
if ("$account" != "") then
echo1 "account: $account"
echo1 "Pbuild_build_directory: $Pbuild_build_directory"
echo1 "Pbuild_install_directory: $Pbuild_install_directory"
endif
echo1 "======================================"
module list >>& $buildinfo
echo1 "======================================"
#===============
# build system
#===============
set cmd1 = "cmake $ESMADIR -DCMAKE_INSTALL_PREFIX=$Pbuild_install_directory -DBASEDIR=${BASEDIR}/${ARCH} -DCMAKE_Fortran_COMPILER=${FORTRAN_COMPILER} -DCMAKE_BUILD_TYPE=${cmake_build_type} -DINSTALL_SOURCE_TARFILE=${INSTALL_SOURCE_TARFILE} ${GMI_MECHANISM_FLAG} ${EXTRA_CMAKE_FLAGS}"
set cmd2 = "make --jobs=$numjobs install $verbose"
echo1 ""
echo1 ""
if ($docmake) then
echo1 "--------------------------------------"
echo1 $cmd1
$cmd1 |& tee -a $cmakelog
echo1 ""
endif
echo1 "--------------------------------------"
echo1 $cmd2
date1
echo1 "Parallel build: $numjobs JOBS on $ncpus CPUs ... "
set echo
$cmd2 |& tee -a $buildlog
set buildstatus = $status
echo1 "build complete; status = $buildstatus"
date1
echo1 "Your build is located in $Pbuild_build_directory"
echo1 ""
echo1 "Your installation is located in $Pbuild_install_directory"
echo1 "You can find setup scripts in $Pbuild_install_directory/bin"
echo1 "--------------------------------------"
time >> $buildinfo
echo1 ""
# check build results
#--------------------
chdir $bldlogdir
ls $buildlog >& /dev/null
set logstatus = $status
exit $buildstatus
usage:
cat <<EOF
usage: $scriptname:t [flagged options]
flagged options
-np do not prompt for responses; use defaults
-help (or -h) echo usage information
-develop checkout with GEOSgcm_GridComp and GEOSgcm_App develop branches
-debug (or -db) compile with debug flags (-DCMAKE_BUILD_TYPE=Debug)
-builddir dir alternate CMake build directory (relative to $ESMADIR)
-installdir dir alternate CMake install directory (relative to $ESMADIR)
-tmpdir dir alternate Fortran TMPDIR location
-esmadir dir esmadir location
-nocmake do not run cmake (useful for scripting)
-gnu build with gfortran
-wait wait when run as a batch job
-no-tar build with INSTALL_SOURCE_TARFILE=OFF (does not tar up source tarball, default is ON)
-i run interactively rather than queuing job
-q qos/queue send batch job to qos/queue
-partition partition send batch job to partition (in case SLURM queue not on default compute partition)
-account account send batch job to account
-walltime hh:mm:ss time to use as batch walltime at job submittal
-mil compile on Milan nodes (default at NCCS)
-rom compile on Rome nodes (default at NAS, only at NAS)
-cas compile on Cascade Lake nodes
-sky compile on Skylake nodes (only at NAS)
-bro compile on Broadwell nodes (only at NAS)
-has compile on Haswell nodes (only at NAS)
extra cmake options
To pass in additional CMake options not covered by the above flags,
after all your flags, add -- and then the options. For example:
$scriptname -debug -- -DSTRATCHEM_REDUCED_MECHANISM=ON -DUSE_CODATA_2018_CONSTANTS=ON
and these options will be appended to the CMake command.
NOTE: Once you use --, you cannot use any more flags. All options
after -- will be passed to CMake and if not a valid CMake option,
could cause the build to fail.
EOF