diff --git a/sorc/build_all.sh b/sorc/build_all.sh index 19f1ae6c92..3c9dc067c7 100755 --- a/sorc/build_all.sh +++ b/sorc/build_all.sh @@ -16,13 +16,15 @@ function _usage() { Builds all of the global-workflow components by calling the individual build scripts in sequence. -Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-v] +Usage: ${BASH_SOURCE[0]} [-a UFS_app][-c build_config][-h][-j n][-v] -a UFS_app: Build a specific UFS app instead of the default -c build_config: Selectively build based on the provided config instead of the default config -h: print this help message and exit + -j: + Specify maximum number of build jobs (n) -v: Execute all build scripts with -v option to turn on verbose where supported EOF @@ -33,25 +35,25 @@ script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd) cd "${script_dir}" || exit 1 _build_ufs_opt="" -_ops_opt="" _verbose_opt="" _partial_opt="" +_build_job_max=20 # Reset option counter in case this script is sourced OPTIND=1 -while getopts ":a:c:hov" option; do +while getopts ":a:c:j:hv" option; do case "${option}" in a) _build_ufs_opt+="-a ${OPTARG} ";; c) _partial_opt+="-c ${OPTARG} ";; h) _usage;; - o) _ops_opt+="-o";; + j) _build_job_max="${OPTARG} ";; v) _verbose_opt="-v";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" - usage + _usage ;; *) echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" - usage + _usage ;; esac done @@ -105,170 +107,189 @@ ERRSCRIPT=${ERRSCRIPT:-'eval [[ $err = 0 ]]'} # shellcheck disable= err=0 -#------------------------------------ -# build gfs_utils -#------------------------------------ -if [[ ${Build_gfs_utils} == 'true' ]]; then - echo " .... Building gfs_utils .... " - # shellcheck disable=SC2086,SC2248 - ./build_gfs_utils.sh ${_verbose_opt} > "${logs_dir}/build_gfs_utils.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building gfs_utils." - echo "The log file is in ${logs_dir}/build_gfs_utils.log" - fi - err=$((err + rc)) -fi +declare -A build_jobs #------------------------------------ -# build WW3 pre & post execs +# Check which builds to do and assign # of build jobs #------------------------------------ -if [[ ${Build_ww3_prepost} == "true" ]]; then - echo " .... Building WW3 pre and post execs .... " - # shellcheck disable=SC2086,SC2248 - ./build_ww3prepost.sh ${_verbose_opt} ${_build_ufs_opt} > "${logs_dir}/build_ww3_prepost.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building WW3 pre/post processing." - echo "The log file is in ${logs_dir}/build_ww3_prepost.log" - fi - err=$((err + rc)) -fi -#------------------------------------ -# build forecast model -#------------------------------------ -if [[ ${Build_ufs_model} == 'true' ]]; then - echo " .... Building forecast model .... " - # shellcheck disable=SC2086,SC2248 - ./build_ufs.sh ${_verbose_opt} ${_build_ufs_opt} > "${logs_dir}/build_ufs.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building UFS model." - echo "The log file is in ${logs_dir}/build_ufs.log" - fi - err=$((err + rc)) -fi +# Mandatory builds, unless otherwise specified, for the UFS +big_jobs=0 +[[ ${Build_ufs_model} == 'true' ]] && build_jobs["ufs_model"]=8 && big_jobs=$((big_jobs+1)) +# The UPP is hardcoded to use 6 cores +[[ ${Build_upp} == 'true' ]] && build_jobs["upp"]=6 +[[ ${Build_ufs_utils} == 'true' ]] && build_jobs["ufs_utils"]=3 +[[ ${Build_gfs_utils} == 'true' ]] && build_jobs["gfs_utils"]=16 +[[ ${Build_ww3prepost} == "true" ]] && build_jobs["ww3prepost"]=2 -#------------------------------------ -# build GSI and EnKF - optional checkout -#------------------------------------ -if [[ -d gsi_enkf.fd ]]; then - if [[ ${Build_gsi_enkf} == 'true' ]]; then - echo " .... Building gsi and enkf .... " - # shellcheck disable=SC2086,SC2248 - ./build_gsi_enkf.sh ${_ops_opt} ${_verbose_opt} > "${logs_dir}/build_gsi_enkf.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building gsi_enkf." - echo "The log file is in ${logs_dir}/build_gsi_enkf.log" - fi - err=$((err + rc)) - fi -else - echo " .... Skip building gsi and enkf .... " -fi +# Optional DA builds +[[ -d gdas.cd ]] && build_jobs["gdas"]=16 && big_jobs=$((big_jobs+1)) +[[ -d gsi_enkf.fd ]] && build_jobs["gsi_enkf"]=8 && big_jobs=$((big_jobs+1)) +[[ -d gsi_utils.fd ]] && build_jobs["gsi_utils"]=2 +[[ -d gsi_monitor.fd ]] && build_jobs["gsi_monitor"]=1 -#------------------------------------ -# build gsi utilities -#------------------------------------ -if [[ -d gsi_utils.fd ]]; then - if [[ ${Build_gsi_utils} == 'true' ]]; then - echo " .... Building gsi utilities .... " - # shellcheck disable=SC2086,SC2248 - ./build_gsi_utils.sh ${_ops_opt} ${_verbose_opt} > "${logs_dir}/build_gsi_utils.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building gsi utilities." - echo "The log file is in ${logs_dir}/build_gsi_utils.log" - fi - err=$((err + rc)) - fi -else - echo " .... Skip building gsi utilities .... " -fi +# Go through all builds and adjust CPU counts down if necessary +requested_cpus=0 +build_list="" +for build in "${!build_jobs[@]}"; do + [[ -z "${build_list}" ]] && build_list="${build}" || build_list="${build_list}, ${build}" + if [[ ${build_jobs[${build}]} -gt ${_build_job_max} ]]; then + build_jobs[${build}]=${_build_job_max} + fi + requested_cpus=$(( requested_cpus + build_jobs[${build}] )) +done -#------------------------------------ -# build gdas - optional checkout -#------------------------------------ -if [[ -d gdas.cd ]]; then - if [[ ${Build_gdas} == 'true' ]]; then - echo " .... Building GDASApp .... " - # shellcheck disable=SC2086,SC2248 - ./build_gdas.sh ${_verbose_opt} > "${logs_dir}/build_gdas.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building GDASApp." - echo "The log file is in ${logs_dir}/build_gdas.log" - fi - err=$((err + rc)) - fi -else - echo " .... Skip building GDASApp .... " -fi +echo "Building ${build_list}" -#------------------------------------ -# build gsi monitor -#------------------------------------ -if [[ -d gsi_monitor.fd ]]; then - if [[ ${Build_gsi_monitor} == 'true' ]]; then - echo " .... Building gsi monitor .... " - # shellcheck disable=SC2086,SC2248 - ./build_gsi_monitor.sh ${_ops_opt} ${_verbose_opt} > "${logs_dir}/build_gsi_monitor.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building gsi monitor." - echo "The log file is in ${logs_dir}/build_gsi_monitor.log" - fi - err=$((err + rc)) - fi -else - echo " .... Skip building gsi monitor .... " +# Go through all builds and adjust CPU counts up if possible +if [[ ${requested_cpus} -lt ${_build_job_max} && ${big_jobs} -gt 0 ]]; then + # Add cores to the gdas, ufs, and gsi build jobs + extra_cores=$(( _build_job_max - requested_cpus )) + extra_cores=$(( extra_cores / big_jobs )) + for build in "${!build_jobs[@]}"; do + if [[ "${build}" == "gdas" || "${build}" == "ufs_model" || "${build}" == "gsi_enkf" ]]; then + build_jobs[${build}]=$(( build_jobs[${build}] + extra_cores )) + fi + done fi -#------------------------------------ -# build UPP -#------------------------------------ -if [[ ${Build_upp} == 'true' ]]; then - echo " .... Building UPP .... " - # shellcheck disable=SC2086,SC2248 - ./build_upp.sh ${_ops_opt} ${_verbose_opt} > "${logs_dir}/build_upp.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building UPP." - echo "The log file is in ${logs_dir}/build_upp.log" - fi - err=$((err + rc)) -fi +procs_in_use=0 +declare -A build_ids -#------------------------------------ -# build ufs_utils -#------------------------------------ -if [[ ${Build_ufs_utils} == 'true' ]]; then - echo " .... Building ufs_utils .... " - # shellcheck disable=SC2086,SC2248 - ./build_ufs_utils.sh ${_verbose_opt} > "${logs_dir}/build_ufs_utils.log" 2>&1 - # shellcheck disable= - rc=$? - if (( rc != 0 )) ; then - echo "Fatal error in building ufs_utils." - echo "The log file is in ${logs_dir}/build_ufs_utils.log" - fi - err=$((err + rc)) -fi +builds_started=0 +# Now start looping through all of the remaining jobs until everything is done +while [[ ${builds_started} -lt ${#build_jobs[@]} ]]; do + if [[ ${procs_in_use} -lt ${_build_job_max} ]]; then + if [[ "${build_jobs[gdas]+0}" && ! "${build_ids[gdas]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['gdas'] + procs_in_use )) ]]; then + ./build_gdas.sh -j "${build_jobs[gdas]}" "${_verbose_opt}" > \ + "${logs_dir}/build_gdas.log" 2>&1 & + build_ids["gdas"]=$! + echo "Starting build_gdas.sh" + procs_in_use=$(( procs_in_use + build_jobs['gdas'] )) + fi + fi + if [[ "${build_jobs[ufs_model]+0}" && ! "${build_ids[ufs_model]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['ufs_model'] + procs_in_use )) ]]; then + ./build_ufs.sh -j "${build_jobs[ufs_model]}" "${_verbose_opt}" "${_build_ufs_opt}" > \ + "${logs_dir}/build_ufs.log" 2>&1 & + build_ids["ufs_model"]=$! + echo "Starting build_ufs.sh" + procs_in_use=$(( procs_in_use + build_jobs['ufs_model'] )) + fi + fi + if [[ "${build_jobs[gsi_enkf]+0}" && ! "${build_ids[gsi_enkf]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['gsi_enkf'] + procs_in_use )) ]]; then + ./build_gsi_enkf.sh -j "${build_jobs[gsi_enkf]}" "${_verbose_opt}" > \ + "${logs_dir}/build_gsi_enkf.log" 2>&1 & + build_ids["gsi_enkf"]=$! + echo "Starting build_gsi_enkf.sh" + procs_in_use=$(( procs_in_use + build_jobs['gsi_enkf'] )) + fi + fi + if [[ "${build_jobs[ufs_utils]+0}" && ! "${build_ids[ufs_utils]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['ufs_utils'] + procs_in_use )) ]]; then + ./build_ufs_utils.sh -j "${build_jobs[ufs_utils]}" "${_verbose_opt}" > \ + "${logs_dir}/build_ufs_utils.log" 2>&1 & + build_ids["ufs_utils"]=$! + echo "Starting build_ufs_utils.sh" + procs_in_use=$(( procs_in_use + build_jobs['ufs_utils'] )) + fi + fi + if [[ "${build_jobs[gsi_utils]+0}" && ! "${build_ids[gsi_utils]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['gsi_utils'] + procs_in_use )) ]]; then + ./build_gsi_utils.sh -j "${build_jobs[gsi_utils]}" "${_verbose_opt}" > \ + "${logs_dir}/build_gsi_utils.log" 2>&1 & + build_ids["gsi_utils"]=$! + echo "Starting build_gsi_utils.sh" + procs_in_use=$(( procs_in_use + build_jobs['gsi_utils'] )) + fi + fi + if [[ "${build_jobs[upp]+0}" && ! "${build_ids[upp]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['upp'] + procs_in_use )) ]]; then + ./build_upp.sh "${_verbose_opt}" > \ + "${logs_dir}/build_upp.log" 2>&1 & + build_ids["upp"]=$! + echo "Starting build_upp.sh" + procs_in_use=$(( procs_in_use + build_jobs['upp'] )) + fi + fi + if [[ "${build_jobs[ww3prepost]+0}" && ! "${build_ids[ww3prepost]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['ww3prepost'] + procs_in_use )) ]]; then + ./build_ww3prepost.sh -j "${build_jobs[ww3prepost]}" "${_verbose_opt}" "${_build_ufs_opt}" > \ + "${logs_dir}/build_ww3prepost.log" 2>&1 & + build_ids["ww3prepost"]=$! + echo "Starting build_ww3prepost.sh" + procs_in_use=$(( procs_in_use + build_jobs['ww3prepost'] )) + fi + fi + if [[ "${build_jobs[gsi_monitor]+0}" && ! "${build_ids[gsi_monitor]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['gsi_monitor'] + procs_in_use )) ]]; then + ./build_gsi_monitor.sh -j "${build_jobs[gsi_monitor]}" "${_verbose_opt}" > \ + "${logs_dir}/build_gsi_monitor.log" 2>&1 & + build_ids["gsi_monitor"]=$! + echo "Starting build_gsi_monitor.sh" + procs_in_use=$(( procs_in_use + build_jobs['gsi_monitor'] )) + fi + fi + if [[ "${build_jobs[gfs_utils]+0}" && ! "${build_ids[gfs_utils]+0}" ]]; then + if [[ ${_build_job_max} -ge $(( build_jobs['gfs_utils'] + procs_in_use )) ]]; then + ./build_gfs_utils.sh -j "${build_jobs[gfs_utils]}" "${_verbose_opt}" > \ + "${logs_dir}/build_gfs_utils.log" 2>&1 & + build_ids["gfs_utils"]=$! + echo "Starting build_gfs_utils.sh" + procs_in_use=$(( procs_in_use + build_jobs['gfs_utils'] )) + fi + fi + fi + + # Check if all builds have completed + # Also recalculate how many processors are in use to account for completed builds + builds_started=0 + procs_in_use=0 + for build in "${!build_jobs[@]}"; do + if [[ "${build_ids[${build}]+0}" ]]; then + builds_started=$(( builds_started + 1)) + # Calculate how many processors are in use + if ps -p "${build_ids[${build}]}" > /dev/null; then + procs_in_use=$(( procs_in_use + build_jobs["${build}"] )) + fi + fi + done + + sleep 5s +done + +# Wait for all jobs to complete and check return statuses +errs=0 +while [[ ${#build_jobs[@]} -gt 0 ]]; do + for build in "${!build_jobs[@]}"; do + # Test if each job is complete and if so, notify and remove from the array + if [[ "${build_ids[${build}]+0}" ]]; then + if ! ps -p "${build_ids[${build}]}" > /dev/null; then + wait "${build_ids[${build}]}" && build_stat=$? + errs=$((errs+build_stat)) + if [[ ${build_stat} == 0 ]]; then + echo "${build} completed successfully!" + else + echo "${build} failed with status ${build_stat}!" + fi + + # Remove the completed build from the list of PIDs + unset 'build_ids[${build}]' + unset 'build_jobs[${build}]' + break + fi + fi + done + + sleep 5s +done #------------------------------------ # Exception Handling #------------------------------------ -if (( err != 0 )); then +if (( errs != 0 )); then cat << EOF BUILD ERROR: One or more components failed to build Check the associated build log(s) for details. diff --git a/sorc/build_gdas.sh b/sorc/build_gdas.sh index 39cf5ac9a7..b1a17c33dd 100755 --- a/sorc/build_gdas.sh +++ b/sorc/build_gdas.sh @@ -2,9 +2,10 @@ set -eux OPTIND=1 -while getopts ":dov" option; do +while getopts ":j:dv" option; do case "${option}" in d) export BUILD_TYPE="DEBUG";; + j) export BUILD_JOBS=${OPTARG};; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" diff --git a/sorc/build_gfs_utils.sh b/sorc/build_gfs_utils.sh index 2a7a611239..09bd4a9656 100755 --- a/sorc/build_gfs_utils.sh +++ b/sorc/build_gfs_utils.sh @@ -5,11 +5,13 @@ function usage() { cat << EOF Builds the GFS utility programs. -Usage: ${BASH_SOURCE[0]} [-d][-h][-v] +Usage: ${BASH_SOURCE[0]} [-d][-h][-j n][-v] -d: Build with debug options -h: Print this help message and exit + -j: + Build with n build jobs -v: Turn on verbose output EOF @@ -19,10 +21,11 @@ EOF cwd=$(pwd) OPTIND=1 -while getopts ":dvh" option; do +while getopts ":j:dvh" option; do case "${option}" in d) export BUILD_TYPE="DEBUG";; v) export BUILD_VERBOSE="YES";; + j) export BUILD_JOBS="${OPTARG}";; h) usage ;; @@ -40,6 +43,7 @@ shift $((OPTIND-1)) BUILD_TYPE=${BUILD_TYPE:-"Release"} \ BUILD_VERBOSE=${BUILD_VERBOSE:-"NO"} \ +BUILD_JOBS=${BUILD_JOBS:-8} \ "${cwd}/gfs_utils.fd/ush/build.sh" exit diff --git a/sorc/build_gsi_enkf.sh b/sorc/build_gsi_enkf.sh index 671c3d6205..9ba278e3ec 100755 --- a/sorc/build_gsi_enkf.sh +++ b/sorc/build_gsi_enkf.sh @@ -2,10 +2,10 @@ set -eux OPTIND=1 -while getopts ":dov" option; do +while getopts ":j:dv" option; do case "${option}" in d) export BUILD_TYPE="DEBUG";; - o) _ops="YES";; + j) export BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" @@ -21,6 +21,7 @@ shift $((OPTIND-1)) BUILD_TYPE=${BUILD_TYPE:-"Release"} \ BUILD_VERBOSE=${BUILD_VERBOSE:-"NO"} \ +BUILD_JOBS=${BUILD_JOBS:-8} \ GSI_MODE=GFS \ ENKF_MODE=GFS \ REGRESSION_TESTS=NO \ diff --git a/sorc/build_gsi_monitor.sh b/sorc/build_gsi_monitor.sh index ec3645e52f..3de1262aac 100755 --- a/sorc/build_gsi_monitor.sh +++ b/sorc/build_gsi_monitor.sh @@ -4,10 +4,10 @@ set -eux cwd=$(pwd) OPTIND=1 -while getopts ":dov" option; do +while getopts ":j:dv" option; do case "${option}" in d) export BUILD_TYPE="DEBUG";; - o) _ops="YES";; + j) export BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" @@ -23,6 +23,7 @@ shift $((OPTIND-1)) BUILD_TYPE=${BUILD_TYPE:-"Release"} \ BUILD_VERBOSE=${BUILD_VERBOSE:-"NO"} \ +BUILD_JOBS=${BUILD_JOBS:-8} \ "${cwd}/gsi_monitor.fd/ush/build.sh" exit diff --git a/sorc/build_gsi_utils.sh b/sorc/build_gsi_utils.sh index bcbc110cf6..81eab0f628 100755 --- a/sorc/build_gsi_utils.sh +++ b/sorc/build_gsi_utils.sh @@ -4,10 +4,10 @@ set -eux cwd=$(pwd) OPTIND=1 -while getopts ":dov" option; do +while getopts ":j:dv" option; do case "${option}" in d) export BUILD_TYPE="DEBUG";; - o) _ops="YES";; # TODO - unused; remove? + j) export BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" @@ -23,6 +23,7 @@ shift $((OPTIND-1)) BUILD_TYPE=${BUILD_TYPE:-"Release"} \ BUILD_VERBOSE=${BUILD_VERBOSE:-"NO"} \ +BUILD_JOBS=${BUILD_JOBS:-8} \ UTIL_OPTS="-DBUILD_UTIL_ENKF_GFS=ON -DBUILD_UTIL_NCIO=ON" \ "${cwd}/gsi_utils.fd/ush/build.sh" diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index aa7183c006..aaeeefea23 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -7,10 +7,11 @@ cwd=$(pwd) APP="S2SWA" CCPP_SUITES="FV3_GFS_v17_p8,FV3_GFS_v17_coupled_p8" # TODO: does the g-w need to build with all these CCPP_SUITES? -while getopts ":da:v" option; do +while getopts ":da:j:v" option; do case "${option}" in d) BUILD_TYPE="DEBUG";; - a) APP="${OPTARG}" ;; + a) APP="${OPTARG}";; + j) BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" @@ -38,21 +39,23 @@ if [[ "${MACHINE_ID}" != "noaacloud" ]]; then mv "./tests/modules.fv3_${COMPILE_NR}.lua" ./tests/modules.ufs_model.lua cp "./modulefiles/ufs_common.lua" ./tests/ufs_common.lua else - + if [[ "${PW_CSP:-}" == "aws" ]]; then + set +x # TODO: This will need to be addressed further when the EPIC stacks are available/supported. module use /contrib/spack-stack/envs/ufswm/install/modulefiles/Core module load stack-intel module load stack-intel-oneapi-mpi module load ufs-weather-model-env/1.0.0 - # TODO: It is still uncertain why this is the only module that is + # TODO: It is still uncertain why this is the only module that is # missing; check the spack build as this needed to be added manually. module load w3emc/2.9.2 # TODO: This has similar issues for the EPIC stack. module list + set -x fi export CMAKE_FLAGS="${MAKE_OPT}" - ./build.sh + BUILD_JOBS=${BUILD_JOBS:-8} ./build.sh mv "${cwd}/ufs_model.fd/build/ufs_model" "${cwd}/ufs_model.fd/tests/ufs_model.x" fi diff --git a/sorc/build_ufs_utils.sh b/sorc/build_ufs_utils.sh index 5e2edf0737..5eabdea471 100755 --- a/sorc/build_ufs_utils.sh +++ b/sorc/build_ufs_utils.sh @@ -1,10 +1,31 @@ #! /usr/bin/env bash set -eux +OPTIND=1 +while getopts ":j:dv" option; do + case "${option}" in + j) export BUILD_JOBS="${OPTARG}";; + o) _ops="YES";; + v) export BUILD_VERBOSE="YES";; + :) + echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" + usage + ;; + *) + echo "[${BASH_SOURCE[0]}]: Unrecognized option: ${option}" + usage + ;; + esac +done +shift $((OPTIND-1)) + script_dir=$(dirname "${BASH_SOURCE[0]}") cd "${script_dir}/ufs_utils.fd" || exit 1 -CMAKE_OPTS="-DGFS=ON" ./build_all.sh +CMAKE_OPTS="-DGFS=ON" \ +BUILD_JOBS=${BUILD_JOBS:-8} \ +BUILD_VERBOSE=${BUILD_VERBOSE:-} \ +./build_all.sh exit diff --git a/sorc/build_ww3prepost.sh b/sorc/build_ww3prepost.sh index 7024a7bba1..ec090bd9b0 100755 --- a/sorc/build_ww3prepost.sh +++ b/sorc/build_ww3prepost.sh @@ -7,9 +7,10 @@ cd "${script_dir}" || exit 1 # Default settings APP="S2SWA" -while getopts "a:v" option; do +while getopts ":j:a:v" option; do case "${option}" in - a) APP="${OPTARG}" ;; + a) APP="${OPTARG}";; + j) BUILD_JOBS="${OPTARG}";; v) export BUILD_VERBOSE="YES";; :) echo "[${BASH_SOURCE[0]}]: ${option} requires an argument" @@ -85,7 +86,7 @@ if (( rc != 0 )); then echo "Fatal error in cmake." exit "${rc}" fi -make -j 8 +make -j "${BUILD_JOBS:-8}" rc=$? if (( rc != 0 )); then echo "Fatal error in make." diff --git a/sorc/checkout.sh b/sorc/checkout.sh index a756c8d040..de4fcdf838 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -151,7 +151,7 @@ source "${topdir}/../workflow/gw_setup.sh" # The checkout version should always be a speciifc commit (hash or tag), not a branch errs=0 # Checkout UFS submodules in parallel -checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-68050e5}" "8" ; errs=$((errs + $?)) +checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-68050e5}" "8" & # Run all other checkouts simultaneously with just 1 core each to handle submodules. checkout "wxflow" "https://github.com/NOAA-EMC/wxflow" "528f5ab" & diff --git a/sorc/gfs_build.cfg b/sorc/gfs_build.cfg index 4dd9b1b74e..8c171072d0 100644 --- a/sorc/gfs_build.cfg +++ b/sorc/gfs_build.cfg @@ -2,7 +2,7 @@ # ***** configuration of global-workflow build ***** Building ufs_model (ufs_model) ........................ yes - Building ww3_prepost (ww3_prepost) .................... yes + Building ww3prepost (ww3prepost) ...................... yes Building gsi_enkf (gsi_enkf) .......................... yes Building gsi_utils (gsi_utils) ........................ yes Building gsi_monitor (gsi_monitor) .................... yes diff --git a/sorc/partial_build.sh b/sorc/partial_build.sh index 64f1a705ec..34b8b557ce 100755 --- a/sorc/partial_build.sh +++ b/sorc/partial_build.sh @@ -3,11 +3,10 @@ # define the array of the name of build program # declare -a Build_prg=("Build_ufs_model" \ - "Build_ww3_prepost" \ + "Build_ww3prepost" \ "Build_gsi_enkf" \ "Build_gsi_utils" \ "Build_gsi_monitor" \ - "Build_ww3_prepost" \ "Build_gdas" \ "Build_upp" \ "Build_ufs_utils" \