From 5c304b513e7b711df9e8a799690dee5a1789fef1 Mon Sep 17 00:00:00 2001
From: Wei Huang <wei.huang@noaa.gov>
Date: Wed, 4 Sep 2024 11:13:59 -0600
Subject: [PATCH] Support global-workflow GEFS C48 on Google Cloud (#2861)

# Description

Support global-worflow GEFS C48 on Google Cloud.

Make env. var. and yaml file changes, so global-workflow GEFS C48 case
can run properly on Google Cloud.

Resolves #2860
---
 env/GOOGLEPW.env                           | 52 ++++++++++++++++++++++
 parm/config/gefs/config.base               |  6 +--
 parm/config/gefs/config.resources          |  4 ++
 parm/config/gefs/config.resources.GOOGLEPW | 11 +++++
 workflow/hosts/googlepw.yaml               |  2 +-
 5 files changed, 71 insertions(+), 4 deletions(-)
 create mode 100644 parm/config/gefs/config.resources.GOOGLEPW

diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env
index f5582ccd4d..7d912eaf8b 100755
--- a/env/GOOGLEPW.env
+++ b/env/GOOGLEPW.env
@@ -43,6 +43,17 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,4 +63,45 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
 fi
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
index 189b7ba446..d601b532cd 100644
--- a/parm/config/gefs/config.base
+++ b/parm/config/gefs/config.base
@@ -345,9 +345,9 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arc
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
-# The tracker, genesis, and METplus jobs are not supported on AWS yet
-# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general.
-if [[ "${machine}" == "AWSPW" ]]; then
+# The tracker, genesis, and METplus jobs are not supported on CSPs yet
+# TODO: we should place these in workflow/hosts/[csp]pw.yaml as part of AWS/AZURE/GOOGLE setup, not for general.
+if [[ "${machine}" =~ "PW" ]]; then
    export DO_WAVE="NO"
 fi
 
diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
index 5667e5efa4..79f3426f56 100644
--- a/parm/config/gefs/config.resources
+++ b/parm/config/gefs/config.resources
@@ -43,6 +43,10 @@ case ${machine} in
     export PARTITION_BATCH="compute"
     max_tasks_per_node=36
     ;;
+  "GOOGLEPW")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=32
+    ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"
     exit 2
diff --git a/parm/config/gefs/config.resources.GOOGLEPW b/parm/config/gefs/config.resources.GOOGLEPW
new file mode 100644
index 0000000000..21e54013c7
--- /dev/null
+++ b/parm/config/gefs/config.resources.GOOGLEPW
@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# GOOGLE-specific job resources
+
+export is_exclusive="True"
+unset memory
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done
diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml
index 38180dd750..2bd9439d5f 100644
--- a/workflow/hosts/googlepw.yaml
+++ b/workflow/hosts/googlepw.yaml
@@ -18,7 +18,7 @@ CHGRP_RSTPROD: 'YES'
 CHGRP_CMD: 'chgrp rstprod' # TODO: This is not yet supported.
 HPSSARCH: 'NO'
 HPSS_PROJECT: emc-global #TODO: See `ATARDIR` below.
-BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR/prototype_ICs'
+BASE_IC: '/bucket/global-workflow-shared-data/ICSDIR'
 LOCALARCH: 'NO'
 ATARDIR: '' # TODO: This will not yet work from GOOGLE.
 MAKE_NSSTBUFR: 'NO'