[develop] Fixes for Issue #608, #610, and #616 (#609)

1) Fix the bad WE2E test configuration file for MET_verification_only_vx (Issue #608).
2) Make creation of symlinks to pregenerated files depend on whether downstream tasks need those symlinks (Issue #610).
3) Set default value of FIXdir to HOMEdir/fix only when RUN_ENVIR="nco", not when RUN_TASK_MAKE_GRID=False; otherwise, set FIXdir to EXPTDIR (Issue #616).
4) Add a flag to the script get_expts_status.sh so that if an experiment hasn't been launched yet, it calls the launch script launch_FV3LAM_wflow.sh to launch it instead of only outputting a message that it's not yet launched.

tests/WE2E/get_expts_status.sh

@@ -79,6 +79,7 @@ Usage:
 
   ${scrfunc_fn} \\
     expts_basedir=\"...\" \\
+    [launch_wflows=\"...\"] \\
     [num_log_lines=\"...\"] \\
     [verbose=\"...\"]
 
@@ -89,6 +90,11 @@ expts_basedir:
 Full path to the experiments base directory, i.e. the directory containing 
 the experiment subdirectories.
 
+launch_wflows:
+Optional flag that determines whether each experiment's workflow should
+be launched if hasn't already.  Should be set to \"TRUE\" or \"FALSE\".
+Default is \"FALSE\".
+
 num_log_lines:
 Optional integer specifying the number of lines from the end of the 
 workflow launch log file (log.launch_FV3LAM_wflow) of each test to 
@@ -123,27 +129,29 @@ fi
 #
 valid_args=( \
   "expts_basedir" \
+  "launch_wflows" \
   "num_log_lines" \
   "verbose" \
   )
 process_args valid_args "$@"
 #
 #-----------------------------------------------------------------------
 #
-# Set the default value of "num_log_lines".
+# Default values for various input arguments.
 #
 #-----------------------------------------------------------------------
 #
+launch_wflows=${launch_wflows:-"FALSE"}
 num_log_lines=${num_log_lines:-"40"}
+verbose=${verbose:-"FALSE"}
 #
 #-----------------------------------------------------------------------
 #
-# Make the default value of "verbose" "FALSE".  Then make sure "verbose"
-# is set to a valid value.
+# Make sure "launch_wflows" and "verbose" have valid values.
 #
 #-----------------------------------------------------------------------
 #
-verbose=${verbose:-"FALSE"}
+launch_wflows=$(boolify "${launch_wflows}")
 verbose=$(boolify "$verbose")
 #
 #-----------------------------------------------------------------------
@@ -396,6 +404,7 @@ for (( i=0; i<=$((num_expts-1)); i++ )); do
 $separator
 Checking workflow status of experiment \"${expt_subdir}\" ..."
   print_info_msg "$msg"
+  print_info_msg "$msg" >> "${expts_status_fp}"
 #
 # Change location to the experiment subdirectory, and check the launch
 # log file for status
@@ -424,18 +433,33 @@ ${wflow_status}
 The last ${num_log_lines} lines of the workflow launch log file 
 (\"${launch_wflow_log_fn}\") are:
 "
-    print_info_msg "$msg" >> "${expts_status_fp}"
     tail -n ${num_log_lines} ${launch_wflow_log_fn} >> "${expts_status_fp}" 
+#
+# If a log file from the launch script is not present in the experiment
+# directory, it means the workflow has not been launched.  In this case,
+# print out an appropriate message.  Then, if launch_wflows is set to 
+# TRUE, launch the workflow and print out further info.
+#
   else
+
     wflow_status="Workflow status:  NOT LAUNCHED YET"
+    if [ "${launch_wflows}" = "TRUE" ]; then
+      wflow_status=${wflow_status}"
+Launching workflow using script \"${launch_wflow_fn}\"..."
+    fi
+
     print_info_msg "${wflow_status}"
     print_info_msg "\
 $separator
 "
-    msg=$msg"
-${wflow_status}
+
+    msg="${wflow_status}
 "
     print_info_msg "$msg" >> "${expts_status_fp}"
+    if [ "${launch_wflows}" = "TRUE" ]; then
+      ./${launch_wflow_fn} >> "${expts_status_fp}" 2>&1
+    fi
+
   fi
 #
 # Change location back to the experiments base directory.

tests/WE2E/test_configs/verification/config.MET_verification_only_vx.yaml

@@ -4,14 +4,17 @@ metadata:
     can successfully run the MET deterministic verification tasks but
     using staged observation and forecast files (i.e. with other tasks
     such as pre-processing, forecast, and post-processing deactivated).
+
 user:
   RUN_ENVIR: community
+
 workflow:
   PREDEF_GRID_NAME: RRFS_CONUS_25km
   DATE_FIRST_CYCL: '2019061500'
   DATE_LAST_CYCL: '2019061500'
   FCST_LEN_HRS: 6
   PREEXISTING_DIR_METHOD: rename
+
 workflow_switches:
 #
 # This test assumes that the post-processed forecast files are staged
@@ -33,7 +36,7 @@ workflow_switches:
 # the GET_OBS_... tasks.  Note we do not specify the obs staging directories
 # (CCPA_OBS_DIR, MRMS_OBS_DIR, and NDAS_OBS_DIR) because those will be
 # automatically set (in a platform-dependent way using the machine file)
-# by the script that runs this test.
+# by the script that runs the WE2E tests.
 #
   RUN_TASK_GET_OBS_CCPA: false
   RUN_TASK_GET_OBS_MRMS: false
@@ -43,11 +46,13 @@ workflow_switches:
 #
   RUN_TASK_VX_GRIDSTAT: true
   RUN_TASK_VX_POINTSTAT: true
+
 #
 # In the "verification" section below, we don't explicitly set the location 
-# of the staged forecast files (specified by VX_FCST_INPUT_BASEDIR) because
-# this location gets set automatically (in a platform-dependent way using
-# the appropriate machine file) by the script that runs this test.
+# of the staged forecast files (via VX_FCST_INPUT_BASEDIR) because this
+# location gets set automatically by the script that runs the WE2E tests.
+# The script sets the location in a platform-dependent way using the
+# appropriate machine file.
 #
 verification:
   VX_FCST_MODEL_NAME: FV3_GFS_v15p2_CONUS_25km

ush/config_defaults.yaml

@@ -672,7 +672,7 @@ workflow:
   #
   #-----------------------------------------------------------------------
   #
-  FIXdir: '{{ EXPTDIR if workflow_switches.RUN_TASK_MAKE_GRID else [user.HOMEdir, "fix"]|path_join }}'
+  FIXdir: '{{ EXPTDIR if user.RUN_ENVIR == "community" else [user.HOMEdir, "fix"]|path_join }}'
   FIXam: '{{ [FIXdir, "fix_am"]|path_join }}'
   FIXclim: '{{ [FIXdir, "fix_clim"]|path_join }}'
   FIXlam: '{{ [FIXdir, "fix_lam"]|path_join }}'
@@ -1474,7 +1474,7 @@ task_get_extrn_ics:
   # the external model started than when the FV3 forecast configured here
   # should start. For example, the forecast should start from a 6 hour
   # forecast of the GFS, then EXTRN_MDL_ICS_OFFSET_HRS=6.
-
+  #
   # FV3GFS_FILE_FMT_ICS:
   # If using the FV3GFS model as the source of the ICs (i.e. if EXTRN_MDL_NAME_ICS
   # is set to "FV3GFS"), this variable specifies the format of the model

ush/setup.py

@@ -419,6 +419,11 @@ def setup(USHdir, user_config_fn="config.yaml", debug: bool = False):
     # -----------------------------------------------------------------------
     #
 
+    workflow_switches = expt_config["workflow_switches"]
+    run_task_make_grid = workflow_switches['RUN_TASK_MAKE_GRID']
+    run_task_make_orog = workflow_switches['RUN_TASK_MAKE_OROG']
+    run_task_make_sfc_climo = workflow_switches['RUN_TASK_MAKE_SFC_CLIMO']
+
     # Necessary tasks are turned on
     pregen_basedir = expt_config["platform"].get("DOMAIN_PREGEN_BASEDIR")
     if pregen_basedir is None and not (
@@ -1068,7 +1073,6 @@ def get_location(xcs, fmt, expt_cfg):
     #
     # -----------------------------------------------------------------------
     #
-    workflow_switches = expt_config["workflow_switches"]
 
     # Ensemble verification can only be run in ensemble mode
     do_ensemble = global_sect["DO_ENSEMBLE"]
@@ -1100,14 +1104,30 @@ def get_location(xcs, fmt, expt_cfg):
     # turned off. Link the files, and check that they all contain the
     # same resolution input.
     #
+    run_task_make_ics = workflow_switches['RUN_TASK_MAKE_LBCS']
+    run_task_make_lbcs = workflow_switches['RUN_TASK_MAKE_ICS']
+    run_task_run_fcst = workflow_switches['RUN_TASK_RUN_FCST']
+    run_task_makeics_or_makelbcs_or_runfcst = run_task_make_ics or \
+                                              run_task_make_lbcs or \
+                                              run_task_run_fcst
+    # Flags for creating symlinks to pre-generated grid, orography, and sfc_climo files.
+    # These consider dependencies of other tasks on each pre-processing task.
+    create_symlinks_to_pregen_files = {
+      "GRID": (not workflow_switches['RUN_TASK_MAKE_GRID']) and \
+              (run_task_make_orog or run_task_make_sfc_climo or run_task_makeics_or_makelbcs_or_runfcst),
+      "OROG": (not workflow_switches['RUN_TASK_MAKE_OROG']) and \
+              (run_task_make_sfc_climo or run_task_makeics_or_makelbcs_or_runfcst),
+      "SFC_CLIMO": (not workflow_switches['RUN_TASK_MAKE_SFC_CLIMO']) and \
+                   (run_task_make_ics or run_task_make_lbcs),
+    }
+
     prep_tasks = ["GRID", "OROG", "SFC_CLIMO"]
     res_in_fixlam_filenames = None
     for prep_task in prep_tasks:
         res_in_fns = ""
-        switch = f"RUN_TASK_MAKE_{prep_task}"
         # If the user doesn't want to run the given task, link the fix
         # file from the staged files.
-        if not workflow_switches[switch]:
+        if create_symlinks_to_pregen_files[prep_task]:
             sect_key = f"task_make_{prep_task.lower()}"
             dir_key = f"{prep_task}_DIR"
             task_dir = expt_config[sect_key].get(dir_key)
@@ -1117,7 +1137,7 @@ def get_location(xcs, fmt, expt_cfg):
                 expt_config[sect_key][dir_key] = task_dir
                 msg = dedent(
                     f"""
-                   {dir_key} will use pre-generated files.
+                   {dir_key} will point to a location containing pre-generated files.
                    Setting {dir_key} = {task_dir}
                    """
                 )