[develop] Allow WE2E run script to set location of staged forecast fi…

…les for verification (#605)

This PR allows the SRW App to run the WE2E test that performs deterministic verification using staged forecast files (MET_verification_only_vx) using files in the authoritative location on a given platform (instead of in a user's directory). This is done by:

1) Specifying the location of the staged forecast files in the machine file (for this PR, it is specified only for Hera) via the new variable TEST_VX_FCST_INPUT_BASEDIR.
2) Modifying run_WE2E_tests.py so that it correctly sets the variable containing this location (VX_FCST_INPUT_BASEDIR). Depending on the experiment configuration, VX_FCST_INPUT_BASEDIR may be set using the value in the test configuration file, the machine file, or the default experiment configuration file.
3) Modifying the configuration file for the WE2E test MET_verification_only_vx so that it does not specify a user directory for VX_FCST_INPUT_BASEDIR (as well as for CCPA_OBS_DIR, MRMS_OBS_DIR, and NDAS_OBS_DIR).

tests/WE2E/run_WE2E_tests.py

@@ -173,6 +173,10 @@ def run_we2e_tests(homedir, args) -> None:
             test_cfg['task_get_extrn_lbcs'] = check_task_get_extrn_lbcs(test_cfg,machine_defaults,config_defaults)
             logging.debug(test_cfg['task_get_extrn_lbcs'])
 
+        if 'verification' in test_cfg:
+            logging.debug(test_cfg['verification'])
+        test_cfg['verification'] = check_task_verification(test_cfg,machine_defaults,config_defaults)
+        logging.debug(test_cfg['verification'])
 
         logging.debug(f"Writing updated config.yaml for test {test_name}\nbased on specified command-line arguments:\n")
         logging.debug(cfg_to_yaml_str(test_cfg))
@@ -395,6 +399,53 @@ def check_task_get_extrn_lbcs(cfg: dict, mach: dict, dflt: dict) -> dict:
 
     return cfg_lbcs
 
+def check_task_verification(cfg: dict, mach: dict, dflt: dict) -> dict:
+    """
+    Function for checking and updating various settings in verification section of test config yaml
+
+    Args:
+        cfg  : Dictionary loaded from test config file
+        mach : Dictionary loaded from machine settings file
+        dflt : Dictionary loaded from default config file
+    Returns:
+        cfg_vx : Updated dictionary for verification section of test config
+    """
+
+    # Make our lives easier by shortening some dictionary calls
+    if 'verification' in cfg:
+        cfg_vx = cfg['verification']
+    else:
+        cfg_vx = dict()
+
+    # If VX_FCST_INPUT_BASEDIR is already explicitly set in the test configuration
+    # dictionary, keep that value and just return.
+    if 'VX_FCST_INPUT_BASEDIR' in cfg_vx:
+        return cfg_vx
+
+    # Attempt to obtain the values of RUN_TASK_RUN_FCST, WRITE_DO_POST, and RUN_TASK_RUN_POST
+    # from the test configuration dictionary.  If not available there, get them from the default 
+    # configuration dictionary.
+    flags = {'RUN_TASK_RUN_FCST': False, 'WRITE_DOPOST': False, 'RUN_TASK_RUN_POST': False}
+    for section in ['workflow_switches', 'task_run_fcst']:
+        for flag in flags:
+            if (section in cfg) and (flag in cfg[section]):
+                flags[flag] = cfg[section][flag]
+            elif flag in dflt[section]:
+                flags[flag] = dflt[section][flag]
+
+    # If UPP is going to be run (either in-line or as a separate set of tasks), set the
+    # VX_FCST_INPUT_BASEDIR to the default directory for the experiment.  Otherwise, set
+    # it to the value of TEST_VX_FCST_INPUT_BASEDIR in the machine file.
+    if (flags['RUN_TASK_RUN_FCST'] and flags['WRITE_DOPOST']) or flags['RUN_TASK_RUN_POST']:
+        cfg_vx['VX_FCST_INPUT_BASEDIR'] = dflt['workflow']['EXPTDIR']
+    else:
+        if 'TEST_VX_FCST_INPUT_BASEDIR' in mach['platform']:
+            cfg_vx['VX_FCST_INPUT_BASEDIR'] = mach['platform']['TEST_VX_FCST_INPUT_BASEDIR']
+        else:
+            raise KeyError(f"Non-default forecast file location for verification (TEST_VX_FCST_INPUT_BASEDIR) not set in machine file")
+
+    return cfg_vx
+
 def setup_logging(logfile: str = "log.run_WE2E_tests", debug: bool = False) -> None:
     """
     Sets up logging, printing high-priority (INFO and higher) messages to screen, and printing all

tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml

@@ -30,22 +30,24 @@ workflow_switches:
   RUN_TASK_RUN_POST: false
 #
 # This test assumes the observation files are staged.  Thus, deactivate
-# the GET_OBS_... tasks and instead specify the obs staging directories.
+# the GET_OBS_... tasks.  Note we do not specify the obs staging directories
+# (CCPA_OBS_DIR, MRMS_OBS_DIR, and NDAS_OBS_DIR) because those will be
+# automatically set (in a platform-dependent way using the machine file)
+# by the script that runs this test.
 #
   RUN_TASK_GET_OBS_CCPA: false
-  CCPA_OBS_DIR: '/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/DTC_ensemble_task/staged/obs/ccpa/proc'
   RUN_TASK_GET_OBS_MRMS: false
-  MRMS_OBS_DIR: '/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/DTC_ensemble_task/staged/obs/mrms/proc'
   RUN_TASK_GET_OBS_NDAS: false
-  NDAS_OBS_DIR: '/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/DTC_ensemble_task/staged/obs/ndas/proc'
 #
 # Turn on verification tasks.
 #
   RUN_TASK_VX_GRIDSTAT: true
   RUN_TASK_VX_POINTSTAT: true
-verification:
-  VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km
 #
-# Since the forecast files are staged, specify the base staging directory.
+# In the "verification" section below, we don't explicitly set the location 
+# of the staged forecast files (specified by VX_FCST_INPUT_BASEDIR) because
+# this location gets set automatically (in a platform-dependent way using
+# the appropriate machine file) by the script that runs this test.
 #
-  VX_FCST_INPUT_BASEDIR: '/scratch2/BMC/det/Gerard.Ketefian/UFS_CAM/DTC_ensemble_task/staged/fcst_det'
+verification:
+  VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km

ush/config_defaults.yaml

@@ -352,6 +352,7 @@ platform:
   TEST_PREGEN_BASEDIR: ""
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: ""
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: ""
+  TEST_VX_FCST_INPUT_BASEDIR: ""
   #
   #-----------------------------------------------------------------------
   #
@@ -484,7 +485,7 @@ workflow:
   EXPT_BASEDIR: '' # This will be set in setup.py prior to extend_yaml() being called
   EXPT_SUBDIR: '{{ EXPT_SUBDIR }}'
   EXEC_SUBDIR: "exec"
-  EXPTDIR: '{{ [EXPT_BASEDIR, EXPT_SUBDIR]|path_join }}'
+  EXPTDIR: '{{ [workflow.EXPT_BASEDIR, workflow.EXPT_SUBDIR]|path_join }}'
   #
   #-----------------------------------------------------------------------
   #
@@ -2604,28 +2605,27 @@ global:
   #-----------------------------------------------------------------------
   #
   HALO_BLEND: 10
+  #
+  #-----------------------------------------------------------------------
+  #
 
 #----------------------------
 # verification parameters
-#----------------------------
+#
+# VX_FCST_MODEL_NAME:
+# String that specifies a descriptive name for the model being verified.
+# This is used in forming the names of the verification output files as
+# well as in the contents of those files.
+#
+# VX_FCST_INPUT_BASEDIR:
+# Location of top-level directory containing forecast (but not obs) files
+# that will be used as input into METplus for verification.  If not
+# specified, this gets set to EXPTDIR.
+#
+#-----------------------------
 verification:
-  #
-  #-----------------------------------------------------------------------
-  #
-  # VX_FCST_MODEL_NAME:
-  # String that specifies a descriptive name for the model being verified.
-  # This is used in forming the names of the verification output files as
-  # well as in the contents of those files.
-  #
-  # VX_FCST_INPUT_BASEDIR:
-  # Location of top-level directory containing forecast (but not obs) files
-  # that will be used as input into METplus for verification.  If not
-  # specified, this gets set to EXPTDIR.
-  #
-  #-----------------------------------------------------------------------
-  #
   VX_FCST_MODEL_NAME: '{{ nco.NET }}.{{ task_run_post.POST_OUTPUT_DOMAIN_NAME }}'
-  VX_FCST_INPUT_BASEDIR: '{{ workflow.EXPTDIR }}'
+  VX_FCST_INPUT_BASEDIR: '{{ workflow.EXPTDIR if ((workflow_switches.RUN_TASK_RUN_FCST and task_run_fcst.WRITE_DOPOST) or workflow_switches.RUN_TASK_RUN_POST) }}'
 
 #----------------------------
 # CPL_AQM config parameters

ush/machine/hera.yaml

@@ -26,13 +26,15 @@ platform:
   TEST_PREGEN_BASEDIR: /scratch2/BMC/det/UFS_SRW_App/develop/FV3LAM_pregen
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /scratch2/BMC/det/UFS_SRW_app/develop/dummy_FV3GFS_sys_dir
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /scratch2/BMC/det/UFS_SRW_app/develop/dummy_FV3GFS_sys_dir
+  TEST_VX_FCST_INPUT_BASEDIR: '/scratch2/BMC/det/UFS_SRW_App/develop/output_data/fcst_det/{{workflow.PREDEF_GRID_NAME}}'
   FIXaer: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_aer
   FIXgsm: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_am
   FIXlut: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_lut
   FIXorg: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_orog
   FIXsfc: /scratch2/BMC/det/UFS_SRW_App/develop/fix/fix_sfc_climo
   FIXshp: /scratch2/BMC/det/UFS_SRW_App/develop/NaturalEarth
   EXTRN_MDL_DATA_STORES: hpss aws nomads
+
 data:
   obs:
     RAP_obs: /scratch2/BMC/public/data/grids/rap/obs