GEFS Staging in exglobal_stage_ic (NOAA-EMC#1892)

Adjusts source paths for forecast-only ICs in support of extension to ensembles.
New paths omit the resolution (that is captured by the IC id name) and allow for
a member level.

Directories with the reorganized ICs have been added to the three tier-1 machines
under glopara space.

Building on this, also adds capability to stage GEFS ICs.

Resolves NOAA-EMC#911

parm/config/gefs/config.base.emc.dyn

@@ -40,6 +40,7 @@ export FIXreg2grb2=${HOMEgfs}/fix/reg2grb2
 export PACKAGEROOT="@PACKAGEROOT@"    # TODO: set via prod_envir in Ops
 export COMROOT="@COMROOT@"    # TODO: set via prod_envir in Ops
 export COMINsyn="@COMINsyn@"
+export BASE_CPLIC="@BASE_CPLIC@"
 
 # USER specific paths
 export HOMEDIR="@HOMEDIR@"

parm/config/gefs/config.resources

@@ -8,7 +8,7 @@ if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input task argument to set resource variables!"
     echo "argument can be any one of the following:"
-    echo "coupled_ic aerosol_init"
+    echo "stage_ic aerosol_init"
     echo "sfcanl analcalc analdiag fcst post vrfy fit2obs metp arch echgres"
     echo "ecen esfc efcs epos earc"
     echo "init_chem mom6ic ocnpost"
@@ -416,12 +416,12 @@ elif [[ ${step} = "arch" || ${step} = "earc" ]]; then
       eval "export memory_${step}=50GB"
     fi
 
-elif [[ ${step} = "coupled_ic" ]]; then
+elif [[ ${step} = "stage_ic" ]]; then
 
-    export wtime_coupled_ic="00:15:00"
-    export npe_coupled_ic=1
-    export npe_node_coupled_ic=1
-    export nth_coupled_ic=1
+    export wtime_stage_ic="00:15:00"
+    export npe_stage_ic=1
+    export npe_node_stage_ic=1
+    export nth_stage_ic=1
     export is_exclusive=True
 
 elif [[ ${step} = "ecen" ]]; then

parm/config/gefs/config.stage_ic

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+
+########## config.stage_ic ##########
+
+echo "BEGIN: config.stage_ic"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" stage_ic
+
+case "${CASE}" in
+  "C48")
+    export CPL_ATMIC="gefs_test"
+    export CPL_ICEIC="gefs_test"
+    export CPL_OCNIC="gefs_test"
+    export CPL_WAVIC="gefs_test"
+    ;;
+  *)
+    echo "FATAL ERROR Unrecognized resolution: ${CASE}"
+    exit 1
+    ;;
+esac
+
+echo "END: config.stage_ic"

parm/config/gfs/config.stage_ic

@@ -9,22 +9,22 @@ source "${EXPDIR}/config.resources" stage_ic
 
 case "${CASE}" in
   "C48" | "C96")
-    export CPL_ATMIC=workflowtest
-    export CPL_ICEIC=workflowtest
-    export CPL_OCNIC=workflowtest
-    export CPL_WAVIC=workflowtest
+    export CPL_ATMIC="workflow_${CASE}_refactored"
+    export CPL_ICEIC="workflow_${CASE}_refactored"
+    export CPL_OCNIC="workflow_${CASE}_refactored"
+    export CPL_WAVIC="workflow_${CASE}_refactored"
     ;;
   "C384")
-    export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c
-    export CPL_ICEIC=CPC
-    export CPL_OCNIC=CPC3Dvar
-    export CPL_WAVIC=GEFSwave20210528v2
+    export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored
+    export CPL_ICEIC=CPC_refactored
+    export CPL_OCNIC=CPC3Dvar_refactored
+    export CPL_WAVIC=GEFSwave20210528v2_refactored
     ;;
   "C768")
-    export CPL_ATMIC=HR2
-    export CPL_ICEIC=HR1
-    export CPL_OCNIC=HR1
-    export CPL_WAVIC=HR1
+    export CPL_ATMIC=HR2_refactored
+    export CPL_ICEIC=HR1_refactored
+    export CPL_OCNIC=HR1_refactored
+    export CPL_WAVIC=HR1_refactored
     ;;
   *)
     echo "FATAL ERROR Unrecognized resolution: ${CASE}"

scripts/exglobal_stage_ic.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
 
@@ -8,99 +8,89 @@ GDATE=$(date --utc -d "${PDY} ${cyc} - ${assim_freq} hours" +%Y%m%d%H)
 gPDY="${GDATE:0:8}"
 gcyc="${GDATE:8:2}"
 
+MEMDIR_ARRAY=()
+if [[ "${RUN}" == "gefs" ]]; then
+  # Populate the member_dirs array based on the value of NMEM_ENS
+  for ((ii = 0; ii <= "${NMEM_ENS}"; ii++)); do
+    MEMDIR_ARRAY+=("mem$(printf "%03d" "${ii}")")
+  done
+else
+  MEMDIR_ARRAY+=("")
+fi
+
 # Initialize return code
 err=0
 
-error_message(){
-    echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})"
+error_message() {
+  echo "FATAL ERROR: Unable to copy ${1} to ${2} (Error code ${3})"
 }
 
 ###############################################################
-# Start staging
-
-# Stage the FV3 initial conditions to ROTDIR (cold start)
-YMD=${PDY} HH=${cyc} generate_com -r COM_ATMOS_INPUT
-[[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
-source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/gfs_ctrl.nc"
-target="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
-${NCP} "${source}" "${target}"
-rc=$?
-(( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
-err=$((err + rc))
-for ftype in gfs_data sfc_data; do
-  for tt in $(seq 1 6); do
-    source="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${CDUMP}/${CASE}/INPUT/${ftype}.tile${tt}.nc"
-    target="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
-    ${NCP} "${source}" "${target}"
-    rc=$?
-    (( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
-    err=$((err + rc))
-  done
-done
-
-# Stage ocean initial conditions to ROTDIR (warm start)
-if [[ "${DO_OCN:-}" = "YES" ]]; then
-  YMD=${gPDY} HH=${gcyc} generate_com -r COM_OCEAN_RESTART
-  [[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}"
-  source="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/ocn/${OCNRES}/MOM.res.nc"
-  target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
-  ${NCP} "${source}" "${target}"
+for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
+  # Stage the FV3 initial conditions to ROTDIR (cold start)
+  YMD=${PDY} HH=${cyc} generate_com COM_ATMOS_INPUT
+  [[ ! -d "${COM_ATMOS_INPUT}" ]] && mkdir -p "${COM_ATMOS_INPUT}"
+  src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/gfs_ctrl.nc"
+  tgt="${COM_ATMOS_INPUT}/gfs_ctrl.nc"
+  ${NCP} "${src}" "${tgt}"
   rc=$?
-  (( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
+  ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
   err=$((err + rc))
-  case "${OCNRES}" in
-    "500" | "100")  # Only 5 degree or 1 degree ocean does not have MOM.res_[1-4].nc files
-    ;;
-    "025")  # Only 1/4 degree ocean has MOM.res_[1-4].nc files
-      for nn in $(seq 1 4); do
-        source="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/ocn/${OCNRES}/MOM.res_${nn}.nc"
-        if [[ -f "${source}" ]]; then
-          target="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res_${nn}.nc"
-          ${NCP} "${source}" "${target}"
-          rc=$?
-          (( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
-          err=$((err + rc))
-        fi
-      done
-    ;;
-    *)
-      echo "FATAL ERROR: Unsupported ocean resolution ${OCNRES}"
-      rc=1
+  for ftype in gfs_data sfc_data; do
+    for ((tt = 1; tt <= 6; tt++)); do
+      src="${BASE_CPLIC}/${CPL_ATMIC}/${PDY}${cyc}/${MEMDIR}/atmos/${ftype}.tile${tt}.nc"
+      tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
+      ${NCP} "${src}" "${tgt}"
+      rc=$?
+      tgt="${COM_ATMOS_INPUT}/${ftype}.tile${tt}.nc"
+      ${NCP} "${src}" "${tgt}"
+      rc=$?
+      ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
       err=$((err + rc))
-    ;;
-  esac
-fi
-
-# Stage ice initial conditions to ROTDIR (warm start)
-if [[ "${DO_ICE:-}" = "YES" ]]; then
-  YMD=${gPDY} HH=${gcyc} generate_com -r COM_ICE_RESTART
-  [[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}"
-  ICERESdec=$(echo "${ICERES}" | awk '{printf "%0.2f", $1/100}')
-  source="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/ice/${ICERES}/cice5_model_${ICERESdec}.res_${PDY}${cyc}.nc"
-  target="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
-  ${NCP} "${source}" "${target}"
-  rc=$?
-  (( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
-  err=$((err + rc))
-fi
+    done
+  done
 
-# Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc)
-if [[ "${DO_WAVE:-}" = "YES" ]]; then
-  YMD=${PDY} HH=${cyc} generate_com -r COM_WAVE_RESTART
-  [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
-  for grdID in ${waveGRD}; do  # TODO: check if this is a bash array; if so adjust
-    source="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/wav/${grdID}/${PDY}.${cyc}0000.restart.${grdID}"
-    target="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
-    ${NCP} "${source}" "${target}"
+  # Stage ocean initial conditions to ROTDIR (warm start)
+  if [[ "${DO_OCN:-}" = "YES" ]]; then
+    YMD=${gPDY} HH=${gcyc} generate_com COM_OCEAN_RESTART
+    [[ ! -d "${COM_OCEAN_RESTART}" ]] && mkdir -p "${COM_OCEAN_RESTART}"
+    src="${BASE_CPLIC}/${CPL_OCNIC}/${PDY}${cyc}/${MEMDIR}/ocean/${PDY}.${cyc}0000.MOM.res.nc"
+    tgt="${COM_OCEAN_RESTART}/${PDY}.${cyc}0000.MOM.res.nc"
+    ${NCP} "${src}" "${tgt}"
     rc=$?
-    (( rc != 0 )) && error_message "${source}" "${target}" "${rc}"
+    ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
     err=$((err + rc))
-  done
-fi
+  fi
+  # Stage ice initial conditions to ROTDIR (warm start)
+  if [[ "${DO_ICE:-}" = "YES" ]]; then
+    YMD=${gPDY} HH=${gcyc} generate_com COM_ICE_RESTART
+    [[ ! -d "${COM_ICE_RESTART}" ]] && mkdir -p "${COM_ICE_RESTART}"
+    src="${BASE_CPLIC}/${CPL_ICEIC}/${PDY}${cyc}/${MEMDIR}/ice/${PDY}.${cyc}0000.cice_model.res.nc"
+    tgt="${COM_ICE_RESTART}/${PDY}.${cyc}0000.cice_model.res.nc"
+    ${NCP} "${src}" "${tgt}"
+    rc=$?
+    ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+    err=$((err + rc))
+  fi
+
+  # Stage the WW3 initial conditions to ROTDIR (warm start; TODO: these should be placed in $RUN.$gPDY/$gcyc)
+  if [[ "${DO_WAVE:-}" = "YES" ]]; then
+    YMD=${PDY} HH=${cyc} generate_com COM_WAVE_RESTART
+    [[ ! -d "${COM_WAVE_RESTART}" ]] && mkdir -p "${COM_WAVE_RESTART}"
+    for grdID in ${waveGRD}; do # TODO: check if this is a bash array; if so adjust
+      src="${BASE_CPLIC}/${CPL_WAVIC}/${PDY}${cyc}/${MEMDIR}/wave/${PDY}.${cyc}0000.restart.${grdID}"
+      tgt="${COM_WAVE_RESTART}/${PDY}.${cyc}0000.restart.${grdID}"
+      ${NCP} "${src}" "${tgt}"
+      rc=$?
+      ((rc != 0)) && error_message "${src}" "${tgt}" "${rc}"
+      err=$((err + rc))
+    done
+  fi
 
+done # for MEMDIR in "${MEMDIR_ARRAY[@]}"; do
 ###############################################################
 # Check for errors and exit if any of the above failed
-if  [[ "${err}" -ne 0 ]] ; then
+if [[ "${err}" -ne 0 ]]; then
   echo "FATAL ERROR: Unable to copy ICs from ${BASE_CPLIC} to ${ROTDIR}; ABORT!"
   exit "${err}"
 fi

workflow/applications/gefs.py

@@ -14,7 +14,7 @@ def _get_app_configs(self):
         """
         Returns the config_files that are involved in gefs
         """
-        configs = ['fcst']
+        configs = ['stage_ic', 'fcst']
 
         if self.nens > 0:
             configs += ['efcs']
@@ -32,7 +32,7 @@ def _update_base(base_in):
 
     def get_task_names(self):
 
-        tasks = ['fcst']
+        tasks = ['stage_ic', 'fcst']
 
         if self.nens > 0:
             tasks += ['efcs']

workflow/hosts/hera.yaml

@@ -1,6 +1,6 @@
 BASE_GIT: '/scratch1/NCEPDEV/global/glopara/git'
 DMPDIR: '/scratch1/NCEPDEV/global/glopara/dump'
-BASE_CPLIC: '/scratch1/NCEPDEV/climate/role.ufscpara/IC'
+BASE_CPLIC: '/scratch1/NCEPDEV/global/glopara/data/ICSDIR/prototype_ICs'
 PACKAGEROOT: '/scratch1/NCEPDEV/global/glopara/nwpara'
 COMROOT: '/scratch1/NCEPDEV/global/glopara/com'
 COMINsyn: '${COMROOT}/gfs/prod/syndat'

workflow/hosts/wcoss2.yaml

@@ -1,6 +1,6 @@
 BASE_GIT: '/lfs/h2/emc/global/save/emc.global/git'
 DMPDIR: '/lfs/h2/emc/dump/noscrub/dump'
-BASE_CPLIC: '/lfs/h2/emc/couple/noscrub/Jiande.Wang/IC'
+BASE_CPLIC: '/lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/prototype_ICs'
 PACKAGEROOT: '${PACKAGEROOT:-"/lfs/h1/ops/prod/packages"}'
 COMROOT: '${COMROOT:-"/lfs/h1/ops/prod/com"}'
 COMINsyn: '${COMROOT}/gfs/v16.3/syndat'