[release/public-v2.2.0] Update the SRW v2.2.0 release to allow it to …

…build and run on Hera and to run the AQM WE2E and sample configuration tests: * modulefiles/build_hera_intel.lua - point to the Rocky 8 spack-stack environment, replace nco/4.9.3 with nco/5.0.6, and use cmake/3.23.1 * modulefiles/tasks/hera/aqm_ics.local.lua - replace nco/4.9.3 with nco/5.0.6 * modulefiles/tasks/hera/aqm_lbcs.local.lua - same * modulefiles/tasks/hera/fire_emission.local.lua - same * modulefiles/tasks/hera/nexus_emission.local.lua - same * modulefiles/tasks/hera/nexus_post_split.local.lua - same * modulefiles/tasks/hera/pre_post_stat.local.lua - same * parm/wflow/aqm_prep.yaml - add SCHED_NATIVE_CMD_HPSS to the native entry for task_nexus_gfs_sfc * parm/wflow/verify_pre.yaml - add SCHED_NATIVE_CMD_HPSS to the native entry for task_get_obs_ccpa, task_get_obs_nohrsc, task_get_obs_mrms, and task_get_obs_ndas * scripts/exregional_fire_emission.sh - corrected path to fire file * tests/WE2E/run_WE2E_tests.py - remove aqm section * tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml - update case to 2023111000 - 2023111100 * ush/config.aqm.community.yaml - update case to 2023111000 - 2023111100 * ush/machine/hera.yaml - add SCHED_NATIVE_CMD_HPSS to use service nodes on Hera and update paths for AQM test * ush/machine/jet.yaml - add SCHED_NATIVE_CMD_HPSS to use service nodes on Jet
MichaelLueken · May 9, 2024 · b184d73 · b184d73
1 parent e0ea410
commit b184d73
Show file tree

Hide file tree

Showing 15 changed files with 35 additions and 38 deletions.
diff --git a/modulefiles/build_hera_intel.lua b/modulefiles/build_hera_intel.lua
@@ -8,7 +8,7 @@ whatis([===[Loads libraries needed for building the UFS SRW App on Hera ]===])
 prepend_path("MODULEPATH","/contrib/sutils/modulefiles")
 load("sutils")
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env-rocky8/install/modulefiles/Core")
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
@@ -20,13 +20,13 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
 load(pathJoin("stack-python", stack_python_ver))
 
-cmake_ver=os.getenv("cmake_ver") or "3.20.1"
+cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
 load("srw_common")
 
 load(pathJoin("nccmp", os.getenv("nccmp_ver") or "1.9.0.1"))
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
 load("ufs-pyenv")
 
 setenv("CMAKE_C_COMPILER","mpiicc")

diff --git a/modulefiles/tasks/hera/aqm_ics.local.lua b/modulefiles/tasks/hera/aqm_ics.local.lua
@@ -1,2 +1,2 @@
 load("miniconda_regional_workflow_cmaq")
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
diff --git a/modulefiles/tasks/hera/aqm_lbcs.local.lua b/modulefiles/tasks/hera/aqm_lbcs.local.lua
@@ -1 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
diff --git a/modulefiles/tasks/hera/fire_emission.local.lua b/modulefiles/tasks/hera/fire_emission.local.lua
@@ -1,3 +1,3 @@
 load("hpss")
 load("miniconda_regional_workflow_cmaq")
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
diff --git a/modulefiles/tasks/hera/nexus_emission.local.lua b/modulefiles/tasks/hera/nexus_emission.local.lua
@@ -1,2 +1,2 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
 load("miniconda_regional_workflow_cmaq")
diff --git a/modulefiles/tasks/hera/nexus_post_split.local.lua b/modulefiles/tasks/hera/nexus_post_split.local.lua
@@ -1,2 +1,2 @@
 load("miniconda_regional_workflow_cmaq")
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
diff --git a/modulefiles/tasks/hera/pre_post_stat.local.lua b/modulefiles/tasks/hera/pre_post_stat.local.lua
@@ -1 +1 @@
-load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
diff --git a/parm/wflow/aqm_prep.yaml b/parm/wflow/aqm_prep.yaml
@@ -23,6 +23,7 @@ default_aqm_task: &default_aqm
 task_nexus_gfs_sfc:
   <<: *default_aqm
   command: '&LOAD_MODULES_RUN_TASK_FP; "nexus_gfs_sfc" "&JOBSdir;/JREGIONAL_NEXUS_GFS_SFC"'
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   join: !cycstr '&LOGDIR;/{{ jobname }}_@Y@m@d@H&LOGEXT;'
   memory: 2G

diff --git a/parm/wflow/verify_pre.yaml b/parm/wflow/verify_pre.yaml
@@ -30,6 +30,7 @@ task_get_obs_ccpa:
     OBS_DIR: '&CCPA_OBS_DIR;'
     OBTYPE: 'CCPA'
     FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h  }}{% endfor %}'
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   queue: "&QUEUE_HPSS;"
   walltime: 00:45:00
@@ -42,6 +43,7 @@ task_get_obs_nohrsc:
     OBS_DIR: '&NOHRSC_OBS_DIR;'
     OBTYPE: 'NOHRSC'
     FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h  }}{% endfor %}'
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   queue: "&QUEUE_HPSS;"
   walltime: 00:45:00
@@ -55,6 +57,7 @@ task_get_obs_mrms:
     OBTYPE: 'MRMS'
     VAR: 'REFC RETOP'
     FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h  }}{% endfor %}'
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   queue: "&QUEUE_HPSS;"
   walltime: 00:45:00
@@ -68,6 +71,7 @@ task_get_obs_ndas:
     FHR: '{% for h in range(0, workflow.FCST_LEN_HRS+1) %}{{ " %02d" % h  }}{% endfor %}'
   command: '&LOAD_MODULES_RUN_TASK_FP; "get_obs" "&JOBSdir;/JREGIONAL_GET_VERIF_OBS"'
   queue: "&QUEUE_HPSS;"
+  native: '{% if platform.get("SCHED_NATIVE_CMD_HPSS") %}{{ platform.SCHED_NATIVE_CMD_HPSS }}{% else %}{{ platform.SCHED_NATIVE_CMD}}{% endif %}'
   partition: '{% if platform.get("PARTITION_HPSS") %}&PARTITION_HPSS;{% else %}None{% endif %}'
   walltime: 02:00:00
 

diff --git a/scripts/exregional_fire_emission.sh b/scripts/exregional_fire_emission.sh
@@ -70,8 +70,8 @@ hh_mh1=${CDATE_mh1:8:2}
 aqm_fire_file_fn="${AQM_FIRE_FILE_PREFIX}_${yyyymmdd}_t${hh}z${AQM_FIRE_FILE_SUFFIX}"
 
 # Check if the fire file exists in the designated directory
-if [ -e "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" ]; then
-  cp_vrfy "${DCOMINfire}/${yyyymmdd}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
+if [ -e "${DCOMINfire}/${aqm_fire_file_fn}" ]; then
+  cp_vrfy "${DCOMINfire}/${aqm_fire_file_fn}" "${FIRE_EMISSION_STAGING_DIR}"
 else
   # Copy raw data 
   for ihr in {0..23}; do

diff --git a/tests/WE2E/run_WE2E_tests.py b/tests/WE2E/run_WE2E_tests.py
@@ -224,10 +224,6 @@ def run_we2e_tests(homedir, args) -> None:
                     logging.debug(f'Setting {obvar} = {mach_path} from machine file')
                     test_cfg['platform'][obvar] = mach_path
 
-        if 'cpl_aqm_parm' in test_cfg:
-            test_aqm_input_basedir = machine_defaults['platform']['TEST_AQM_INPUT_BASEDIR']
-            test_cfg['cpl_aqm_parm']['DCOMINfire_default'] = f"{test_aqm_input_basedir}/RAVE_fire"
-
         logging.debug(f"Writing updated config.yaml for test {test_name}\n"\
                        "based on specified command-line arguments:\n")
         logging.debug(cfg_to_yaml_str(test_cfg))

diff --git a/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml b/tests/WE2E/test_configs/aqm/config.aqm_grid_AQM_NA13km_suite_GFS_v16.yaml
@@ -5,13 +5,10 @@ user:
 workflow:
   PREDEF_GRID_NAME: AQM_NA_13km
   CCPP_PHYS_SUITE: FV3_GFS_v16
-  DATE_FIRST_CYCL: '2023021700'
-  DATE_LAST_CYCL: '2023021706'
-  INCR_CYCL_FREQ: 6
-  FCST_LEN_HRS: -1
-  FCST_LEN_CYCL:
-    - 6
-    - 12
+  DATE_FIRST_CYCL: '2023111000'
+  DATE_LAST_CYCL: '2023111100'
+  INCR_CYCL_FREQ: 24
+  FCST_LEN_HRS: 24
   PREEXISTING_DIR_METHOD: rename
   DIAG_TABLE_TMPL_FN: diag_table_aqm.FV3_GFS_v16
   FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16
@@ -24,7 +21,7 @@ rocoto:
     task_aqm_ics_ext:
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
-        walltime: 02:00:00
+        walltime: 01:20:00
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   FV3GFS_FILE_FMT_ICS: netcdf
@@ -41,7 +38,7 @@ task_run_fcst:
   LAYOUT_X: 50
   LAYOUT_Y: 34
   BLOCKSIZE: 16
-  RESTART_INTERVAL: 6
+  RESTART_INTERVAL: 12 24
   QUILTING: true
   PRINT_ESMF: false
   DO_FCST_RESTART: false

diff --git a/ush/config.aqm.community.yaml b/ush/config.aqm.community.yaml
@@ -10,16 +10,13 @@ workflow:
   EXPT_SUBDIR: aqm_community_aqmna13
   PREDEF_GRID_NAME: AQM_NA_13km
   CCPP_PHYS_SUITE: FV3_GFS_v16
-  DATE_FIRST_CYCL: '2023021700'
-  DATE_LAST_CYCL: '2023021706'
-  INCR_CYCL_FREQ: 6
-  FCST_LEN_HRS: -1
-  FCST_LEN_CYCL:
-    - 6
-    - 12
+  DATE_FIRST_CYCL: '2023111000'
+  DATE_LAST_CYCL: '2023111100'
+  INCR_CYCL_FREQ: 24
+  FCST_LEN_HRS: 24
   PREEXISTING_DIR_METHOD: rename
   VERBOSE: true
-  DEBUG: true
+  DEBUG: false
   COMPILER: intel
   DIAG_TABLE_TMPL_FN: diag_table_aqm.FV3_GFS_v16
   FIELD_TABLE_TMPL_FN: field_table_aqm.FV3_GFS_v16
@@ -32,7 +29,7 @@ rocoto:
     task_aqm_ics_ext:
     metatask_run_ensemble:
       task_run_fcst_mem#mem#:
-        walltime: 02:00:00
+        walltime: 01:20:00
 task_get_extrn_ics:
   EXTRN_MDL_NAME_ICS: FV3GFS
   FV3GFS_FILE_FMT_ICS: netcdf
@@ -47,7 +44,7 @@ task_run_fcst:
   LAYOUT_X: 50
   LAYOUT_Y: 34
   BLOCKSIZE: 16
-  RESTART_INTERVAL: 6
+  RESTART_INTERVAL: 12 24
   QUILTING: true
   PRINT_ESMF: false
   DO_FCST_RESTART: false

diff --git a/ush/machine/hera.yaml b/ush/machine/hera.yaml
@@ -20,9 +20,10 @@ platform:
   RUN_CMD_NEXUS: srun -n ${nprocs} --export=ALL
   RUN_CMD_AQMLBC: srun --export=ALL -n ${NUMTS}
   SCHED_NATIVE_CMD: --export=NONE
+  SCHED_NATIVE_CMD_HPSS: -n 1 --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
-  TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/v2p2/input_model_data
-  TEST_AQM_INPUT_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/v2p2/aqm_data
+  TEST_EXTRN_MDL_SOURCE_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/input_model_data
+  TEST_AQM_INPUT_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/aqm_data
   TEST_PREGEN_BASEDIR: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/v2p2/FV3LAM_pregen
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_ICS: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/v2p2/dummy_FV3GFS_sys_dir
   TEST_ALT_EXTRN_MDL_SYSBASEDIR_LBCS: /scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/v2p2/dummy_FV3GFS_sys_dir
@@ -36,8 +37,8 @@ platform:
   EXTRN_MDL_DATA_STORES: hpss aws nomads
 
 cpl_aqm_parm:
-  AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/epa/data
-  DCOMINbio_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/aqm/bio
+  AQM_CONFIG_DIR: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/fix/aqm/epa/data
+  DCOMINbio_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/fix/aqm/bio
   DCOMINdust_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/FENGSHA
   DCOMINcanopy_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/canopy
   DCOMINfire_default: /scratch2/NCEPDEV/naqfc/RRFS_CMAQ/RAVE_fire

diff --git a/ush/machine/jet.yaml b/ush/machine/jet.yaml
@@ -18,6 +18,7 @@ platform:
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL
   SCHED_NATIVE_CMD: --export=NONE
+  SCHED_NATIVE_CMD_HPSS: -n 1 --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/v2p2/input_model_data
   TEST_PREGEN_BASEDIR: /mnt/lfs4/HFIP/hfv3gfs/role.epic/UFS_SRW_data/v2p2/FV3LAM_pregen
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		load(pathJoin("nco", os.getenv("nco_ver") or "4.9.3"))
		load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))