Merge remote-tracking branch 'emc/develop' into fix/early_run_configs

env/GAEA.env

@@ -34,12 +34,42 @@ else
     exit 2
 fi
 
-if [[ "${step}" = "waveinit" ]]; then
+if [[ "${step}" = "prep" ]]; then
 
-    export CFP_MP="YES"
-    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
-    export wavempexec=${launcher}
-    export wave_mpmd=${mpmd_opt}
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="GAEA"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    ntasks_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    ntasks_gausfcanl=${ntasks_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    export NTHREADS_CYCLE=${threads_per_task:-14}
+    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
 
 elif [[ "${step}" = "fcst" ]]; then
 
@@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "upp" ]]; then
+
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"
+
 elif [[ "${step}" = "atmos_products" ]]; then
 
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Gaea
 
 elif [[ "${step}" = "oceanice_products" ]]; then
 

env/GOOGLEPW.env

@@ -43,6 +43,17 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,4 +63,45 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
 fi

modulefiles/module_base.gaea.lua

@@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
 load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
 load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
 load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None")))
 
 load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
 load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
@@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
 load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
 load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
 load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
@@ -36,4 +38,13 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
+--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles"))
+load(pathJoin("prepobs", "1.1.0"))
+
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+
 whatis("Description: GFS run setup environment")

parm/config/gefs/config.base

@@ -345,9 +345,9 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arc
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
-# The tracker, genesis, and METplus jobs are not supported on AWS yet
-# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general.
-if [[ "${machine}" == "AWSPW" ]]; then
+# The tracker, genesis, and METplus jobs are not supported on CSPs yet
+# TODO: we should place these in workflow/hosts/[csp]pw.yaml as part of AWS/AZURE/GOOGLE setup, not for general.
+if [[ "${machine}" =~ "PW" ]]; then
    export DO_WAVE="NO"
 fi
 

parm/config/gefs/config.resources

@@ -43,6 +43,10 @@ case ${machine} in
     export PARTITION_BATCH="compute"
     max_tasks_per_node=36
     ;;
+  "GOOGLEPW")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=32
+    ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"
     exit 2

parm/config/gefs/config.resources.GOOGLEPW

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# GOOGLE-specific job resources
+
+export is_exclusive="True"
+unset memory
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done

versions/run.gaea.ver

@@ -2,5 +2,7 @@ export stack_intel_ver=2023.1.0
 export stack_cray_mpich_ver=8.1.25
 export spack_env=gsi-addon-dev
 
+export perl_ver=5.38.2
+
 source "${HOMEgfs:-}/versions/run.spack.ver"
 export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

workflow/hosts/googlepw.yaml

@@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
-BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs'
+BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from GOOGLE.
 MAKE_NSSTBUFR: 'NO'