Skip to content

Commit

Permalink
sync
Browse files Browse the repository at this point in the history
  • Loading branch information
@weihuang-jedi
weihuang-jedi committed Aug 9, 2024
2 parents e9734a9 + c2b765d commit a96dcca
Show file tree
Hide file tree
Showing 2 changed files with 186 additions and 14 deletions.
198 changes: 185 additions & 13 deletions env/AWSPW.env
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,151 @@ else
exit 2
fi

if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then

export POE="NO"
export BACK="NO"
export sys_tp="HERA"
export launcher_PREP="srun"

elif [[ "${step}" = "prepsnowobs" ]]; then

export APRUN_CALCFIMS="${APRUN}"

elif [[ "${step}" = "prep_emissions" ]]; then

export APRUN="${APRUN}"

elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then

export CFP_MP="YES"
if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
export wavempexec=${launcher}
export wave_mpmd=${mpmd_opt}

elif [[ "${step}" = "atmanlvar" ]]; then

export NTHREADS_ATMANLVAR=${NTHREADSmax}
export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"

elif [[ "${step}" = "atmensanlletkf" ]]; then

export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
export APRUN_ATMENSANLLETKF="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}"

elif [[ "${step}" = "atmensanlfv3inc" ]]; then

export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"

elif [[ "${step}" = "aeroanlrun" ]]; then

export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_AEROANL=${NTHREADSmax}
export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"

elif [[ "${step}" = "atmanlfv3inc" ]]; then

export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"

elif [[ "${step}" = "prepobsaero" ]]; then

export NTHREADS_PREPOBSAERO=${NTHREADS1}
export APRUN_PREPOBSAERO="${APRUN} --cpus-per-task=${NTHREADS_PREPOBSAERO}"

elif [[ "${step}" = "snowanl" ]]; then

export NTHREADS_SNOWANL=${NTHREADSmax}
export APRUN_SNOWANL="${APRUN} --cpus-per-task=${NTHREADS_SNOWANL}"

export APRUN_APPLY_INCR="${launcher} -n 6"

elif [[ "${step}" = "marinebmat" ]]; then

export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
export APRUN_MARINEBMAT="${APRUN}"

elif [[ "${step}" = "marinebmat" ]]; then

export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
export APRUN_MARINEBMAT="${APRUN}"

elif [[ "${step}" = "ocnanalrun" ]]; then

export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

export APRUN_OCNANAL="${APRUN}"

elif [[ "${step}" = "ocnanalchkpt" ]]; then

export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

export APRUN_OCNANAL="${APRUN}"

elif [[ "${step}" = "ocnanalecen" ]]; then

export NTHREADS_OCNANALECEN=${NTHREADSmax}
export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}"

elif [[ "${step}" = "marineanalletkf" ]]; then

export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"

elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

export NTHREADS_GSI=${NTHREADSmax}
export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"

export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
ntasks_cycle=${ntiles:-6}
export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"

export NTHREADS_GAUSFCANL=1
ntasks_gausfcanl=${ntasks_gausfcanl:-1}
export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"

elif [[ "${step}" = "sfcanl" ]]; then

export NTHREADS_CYCLE=${threads_per_task:-14}
export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"

elif [[ "${step}" = "eobs" ]]; then

export MKL_NUM_THREADS=4
export MKL_CBWR=AUTO

export NTHREADS_GSI=${NTHREADSmax}
export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

elif [[ "${step}" = "eupd" ]]; then

export NTHREADS_ENKF=${NTHREADSmax}
export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}"

export CFP_MP=${CFP_MP:-"YES"}
export USE_CFP=${USE_CFP:-"YES"}
export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then

export launcher="srun --mpi=pmi2 -l"

Expand All @@ -52,12 +196,17 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}

elif [[ "${step}" = "post" ]]; then

export NTHREADS_NP=${NTHREADS1}
export APRUN_NP="${APRUN}"
export NTHREADS_UPP=${NTHREADS1}
export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"

elif [[ "${step}" = "atmos_products" ]]; then

export USE_CFP="YES" # Use MPMD for downstream product generation on Hera

elif [[ "${step}" = "oceanice_products" ]]; then

export NTHREADS_DWN=${threads_per_task_dwn:-1}
[[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
export NTHREADS_OCNICEPOST=${NTHREADS1}
export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"

elif [[ "${step}" = "atmos_products" ]]; then

Expand All @@ -71,33 +220,56 @@ elif [[ "${step}" = "oceanice_products" ]]; then
elif [[ "${step}" = "ecen" ]]; then

export NTHREADS_ECEN=${NTHREADSmax}
export APRUN_ECEN="${APRUN}"
export APRUN_ECEN="${APRUN} --cpus-per-task=${NTHREADS_ECEN}"

export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
[[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
export APRUN_CHGRES="time"

export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
[[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
export APRUN_CALCINC="${APRUN}"
export APRUN_CALCINC="${APRUN} --cpus-per-task=${NTHREADS_CALCINC}"

elif [[ "${step}" = "esfc" ]]; then

export NTHREADS_ESFC=${NTHREADSmax}
export APRUN_ESFC="${APRUN}"
export NTHREADS_ESFC=${threads_per_task_esfc:-${max_threads_per_task}}
export APRUN_ESFC="${APRUN} --cpus-per-task=${NTHREADS_ESFC}"

export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
[[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
export APRUN_CYCLE="${APRUN}"
export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"

elif [[ "${step}" = "epos" ]]; then

export NTHREADS_EPOS=${NTHREADSmax}
export APRUN_EPOS="${APRUN}"
export APRUN_EPOS="${APRUN} --cpus-per-task=${NTHREADS_EPOS}"

elif [[ "${step}" = "postsnd" ]]; then

export CFP_MP="YES"

export NTHREADS_POSTSND=${NTHREADS1}
export APRUN_POSTSND="${APRUN} --cpus-per-task=${NTHREADS_POSTSND}"

export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
[[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}
export APRUN_POSTSNDCFP="${launcher} -n ${ntasks_postsndcfp} ${mpmd_opt}"

elif [[ "${step}" = "awips" ]]; then

export NTHREADS_AWIPS=${NTHREADS1}
export APRUN_AWIPSCFP="${APRUN} ${mpmd_opt}"

elif [[ "${step}" = "gempak" ]]; then

export CFP_MP="YES"

export NTHREADS_GEMPAK=${NTHREADS1}
[[ ${NTHREADS_GEMPAK} -gt ${max_threads_per_task} ]] && export NTHREADS_GEMPAK=${max_threads_per_task}

elif [[ "${step}" = "fit2obs" ]]; then

export NTHREADS_FIT2OBS=${NTHREADS1}
export MPIRUN="${APRUN}"
export MPIRUN="${APRUN} --cpus-per-task=${NTHREADS_FIT2OBS}"

fi
2 changes: 1 addition & 1 deletion workflow/hosts/awspw.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ LOCALARCH: 'NO'
ATARDIR: '' # TODO: This will not yet work from AWS.
MAKE_NSSTBUFR: 'NO'
MAKE_ACFTBUFR: 'NO'
SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions.
SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions.

0 comments on commit a96dcca

Please sign in to comment.