diff --git a/jobs/JHAFS_FORECAST b/jobs/JHAFS_FORECAST
index 584a286cc..871dccc3b 100755
--- a/jobs/JHAFS_FORECAST
+++ b/jobs/JHAFS_FORECAST
@@ -21,7 +21,12 @@ if [ "${RUN_ENVIR^^}" != NCO ]; then
module unload intelpython
fi
module use ${HOMEhafs}/sorc/hafs_forecast.fd/modulefiles
- module load ufs_${machine}
+ module unload intelpython
+ if [ "${machine}" = "wcoss_cray" ] || [ "${machine}" = "wcoss_dell_p3" ]; then
+ module load ufs_${machine}
+ else
+ module load ufs_${machine}.intel
+ fi
module list
fi
diff --git a/jobs/JHAFS_HRDGRAPHICS b/jobs/JHAFS_HRDGRAPHICS
index a9f003ee5..406ce8fbe 100755
--- a/jobs/JHAFS_HRDGRAPHICS
+++ b/jobs/JHAFS_HRDGRAPHICS
@@ -47,7 +47,7 @@ export STORMID=${STORMID:-00L}
export ENSDA=${ENSDA:-NO}
export INPdir=${INPdir:-${WORKhafs}/forecast}
-export DATA=${WORKhafs}/hrd_graphics
+export DATA=${WORKhafs}/hrdgraphics
export SENDCOM=${SENDCOM:-YES}
export SCRUBDATA=${SCRUBDATA:-YES}
diff --git a/parm/forecast/globnest/data_table b/parm/forecast/globnest/data_table
deleted file mode 100644
index e69de29bb..000000000
diff --git a/parm/forecast/globnest/input.nml.tmp b/parm/forecast/globnest/input.nml.tmp
index e5fe82409..f8fe1180c 100644
--- a/parm/forecast/globnest/input.nml.tmp
+++ b/parm/forecast/globnest/input.nml.tmp
@@ -1,22 +1,8 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
/
@@ -170,7 +156,6 @@
fhcyc = 24.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 11
pdfcld = .false.
fhswr = _fhswr_
diff --git a/parm/forecast/globnest/input_nest02.nml.tmp b/parm/forecast/globnest/input_nest02.nml.tmp
index 8ffbbde11..3665d3563 100644
--- a/parm/forecast/globnest/input_nest02.nml.tmp
+++ b/parm/forecast/globnest/input_nest02.nml.tmp
@@ -1,22 +1,8 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
/
@@ -159,7 +145,6 @@
fhcyc = 24.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 11
pdfcld = .false.
fhswr = _fhswr_
diff --git a/parm/forecast/globnest/model_configure.tmp b/parm/forecast/globnest/model_configure.tmp
index 2d98bbc73..bf35fa39f 100644
--- a/parm/forecast/globnest/model_configure.tmp
+++ b/parm/forecast/globnest/model_configure.tmp
@@ -1,6 +1,4 @@
print_esmf: .false.
-total_member: 1
-PE_MEMBER01: NTASKS
start_year: YR
start_month: MN
start_day: DY
@@ -8,16 +6,8 @@ start_hour: H_R
start_minute: 0
start_second: 0
nhours_fcst: NHRS
-RUN_CONTINUE: .false.
-ENS_SPS: .false.
dt_atmos: _dt_atmos_
-cpl: _cpl_
calendar: 'julian'
-memuse_verbose: .false.
-atmos_nthreads: NTHRD
-use_hyper_thread: .false.
-ncores_per_node: NCNODE
-debug_affinity: .true.
restart_interval: _restart_interval_
output_1st_tstep_rst: .false.
@@ -36,8 +26,6 @@ jchunk3d: -1
kchunk3d: -1
ideflate: 1
nbits: 0
-write_nemsioflip: .false.
-write_fsyncflag: .false.
app_domain: '_app_domain_'
output_grid: '_OUTPUT_GRID_'
@@ -54,3 +42,4 @@ nfhout: 3
nfhmax_hf: -1
nfhout_hf: 3
nsout: -1
+output_fh: -1
diff --git a/parm/forecast/globnest_hwrf/data_table b/parm/forecast/globnest_hwrf/data_table
deleted file mode 100644
index e69de29bb..000000000
diff --git a/parm/forecast/globnest_hwrf/input.nml.tmp b/parm/forecast/globnest_hwrf/input.nml.tmp
index ae6e2eafa..c5b1e983e 100644
--- a/parm/forecast/globnest_hwrf/input.nml.tmp
+++ b/parm/forecast/globnest_hwrf/input.nml.tmp
@@ -1,22 +1,8 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
/
@@ -169,7 +155,6 @@
fhcyc = 0.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 15 ! F-A MP scheme
RHGRD = 0.975 ! F-A
spec_adv = .true. ! F-A
diff --git a/parm/forecast/globnest_hwrf/input_nest02.nml.tmp b/parm/forecast/globnest_hwrf/input_nest02.nml.tmp
index 3687f4ebd..0fca26752 100644
--- a/parm/forecast/globnest_hwrf/input_nest02.nml.tmp
+++ b/parm/forecast/globnest_hwrf/input_nest02.nml.tmp
@@ -1,22 +1,8 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
/
@@ -159,7 +145,6 @@
fhcyc = 0.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 15 ! F-A MP scheme
RHGRD = 1.0 ! F-A
spec_adv = .true. ! F-A
diff --git a/parm/forecast/globnest_hwrf/model_configure.tmp b/parm/forecast/globnest_hwrf/model_configure.tmp
index 2d98bbc73..bf35fa39f 100644
--- a/parm/forecast/globnest_hwrf/model_configure.tmp
+++ b/parm/forecast/globnest_hwrf/model_configure.tmp
@@ -1,6 +1,4 @@
print_esmf: .false.
-total_member: 1
-PE_MEMBER01: NTASKS
start_year: YR
start_month: MN
start_day: DY
@@ -8,16 +6,8 @@ start_hour: H_R
start_minute: 0
start_second: 0
nhours_fcst: NHRS
-RUN_CONTINUE: .false.
-ENS_SPS: .false.
dt_atmos: _dt_atmos_
-cpl: _cpl_
calendar: 'julian'
-memuse_verbose: .false.
-atmos_nthreads: NTHRD
-use_hyper_thread: .false.
-ncores_per_node: NCNODE
-debug_affinity: .true.
restart_interval: _restart_interval_
output_1st_tstep_rst: .false.
@@ -36,8 +26,6 @@ jchunk3d: -1
kchunk3d: -1
ideflate: 1
nbits: 0
-write_nemsioflip: .false.
-write_fsyncflag: .false.
app_domain: '_app_domain_'
output_grid: '_OUTPUT_GRID_'
@@ -54,3 +42,4 @@ nfhout: 3
nfhmax_hf: -1
nfhout_hf: 3
nsout: -1
+output_fh: -1
diff --git a/parm/forecast/regional/data_table b/parm/forecast/regional/data_table
deleted file mode 100644
index e69de29bb..000000000
diff --git a/parm/forecast/regional/input.nml.tmp b/parm/forecast/regional/input.nml.tmp
index 03c84ceb2..beed44eec 100644
--- a/parm/forecast/regional/input.nml.tmp
+++ b/parm/forecast/regional/input.nml.tmp
@@ -1,24 +1,9 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
- merge_import = _merge_import_
/
&diag_manager_nml
@@ -164,7 +149,6 @@
fhcyc = 24.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 11
pdfcld = .false.
fhswr = _fhswr_
@@ -223,6 +207,7 @@
cplice = .false.
cplwav = .false.
cplwav2atm = .false.
+ cpl_imp_mrg = _merge_import_
psautco = 0.0008, 0.0005
prautco = 0.00015, 0.00015
iau_delthrs = 6
diff --git a/parm/forecast/regional/model_configure.tmp b/parm/forecast/regional/model_configure.tmp
index 2d98bbc73..bf35fa39f 100644
--- a/parm/forecast/regional/model_configure.tmp
+++ b/parm/forecast/regional/model_configure.tmp
@@ -1,6 +1,4 @@
print_esmf: .false.
-total_member: 1
-PE_MEMBER01: NTASKS
start_year: YR
start_month: MN
start_day: DY
@@ -8,16 +6,8 @@ start_hour: H_R
start_minute: 0
start_second: 0
nhours_fcst: NHRS
-RUN_CONTINUE: .false.
-ENS_SPS: .false.
dt_atmos: _dt_atmos_
-cpl: _cpl_
calendar: 'julian'
-memuse_verbose: .false.
-atmos_nthreads: NTHRD
-use_hyper_thread: .false.
-ncores_per_node: NCNODE
-debug_affinity: .true.
restart_interval: _restart_interval_
output_1st_tstep_rst: .false.
@@ -36,8 +26,6 @@ jchunk3d: -1
kchunk3d: -1
ideflate: 1
nbits: 0
-write_nemsioflip: .false.
-write_fsyncflag: .false.
app_domain: '_app_domain_'
output_grid: '_OUTPUT_GRID_'
@@ -54,3 +42,4 @@ nfhout: 3
nfhmax_hf: -1
nfhout_hf: 3
nsout: -1
+output_fh: -1
diff --git a/parm/forecast/regional_hwrf/data_table b/parm/forecast/regional_hwrf/data_table
deleted file mode 100644
index e69de29bb..000000000
diff --git a/parm/forecast/regional_hwrf/input.nml.tmp b/parm/forecast/regional_hwrf/input.nml.tmp
index b5a1da7f3..9b30fb5e9 100644
--- a/parm/forecast/regional_hwrf/input.nml.tmp
+++ b/parm/forecast/regional_hwrf/input.nml.tmp
@@ -1,24 +1,9 @@
- &amip_interp_nml
- interp_oi_sst = .true.
- use_ncep_sst = .true.
- use_ncep_ice = .false.
- no_anom_sst = .false.
- data_set = 'reynolds_oi'
- date_out_of_range = 'climo'
-/
-
&atmos_model_nml
blocksize = _blocksize_
chksum_debug = .false.
dycore_only = .false.
- fdiag = 3
avg_max_length = 3600.
- fhmax = _fhmax_
- fhout = 3
- fhmaxhf = 0
- fhouthf = 3
ccpp_suite = '_ccpp_suite_'
- merge_import = _merge_import_
/
&diag_manager_nml
@@ -164,7 +149,6 @@
fhcyc = 0.
use_ufo = .true.
pre_rad = .false.
- ncld = 5
imp_physics = 15 ! F-A MP scheme
RHGRD = 1.0 ! F-A
spec_adv = .true. ! F-A
@@ -228,6 +212,7 @@
cplice = .false.
cplwav = .false.
cplwav2atm = .false.
+ cpl_imp_mrg = _merge_import_
psautco = 0.0008, 0.0005
prautco = 0.00015, 0.00015
iau_delthrs = 6
diff --git a/parm/forecast/regional_hwrf/model_configure.tmp b/parm/forecast/regional_hwrf/model_configure.tmp
index 2d98bbc73..bf35fa39f 100644
--- a/parm/forecast/regional_hwrf/model_configure.tmp
+++ b/parm/forecast/regional_hwrf/model_configure.tmp
@@ -1,6 +1,4 @@
print_esmf: .false.
-total_member: 1
-PE_MEMBER01: NTASKS
start_year: YR
start_month: MN
start_day: DY
@@ -8,16 +6,8 @@ start_hour: H_R
start_minute: 0
start_second: 0
nhours_fcst: NHRS
-RUN_CONTINUE: .false.
-ENS_SPS: .false.
dt_atmos: _dt_atmos_
-cpl: _cpl_
calendar: 'julian'
-memuse_verbose: .false.
-atmos_nthreads: NTHRD
-use_hyper_thread: .false.
-ncores_per_node: NCNODE
-debug_affinity: .true.
restart_interval: _restart_interval_
output_1st_tstep_rst: .false.
@@ -36,8 +26,6 @@ jchunk3d: -1
kchunk3d: -1
ideflate: 1
nbits: 0
-write_nemsioflip: .false.
-write_fsyncflag: .false.
app_domain: '_app_domain_'
output_grid: '_OUTPUT_GRID_'
@@ -54,3 +42,4 @@ nfhout: 3
nfhmax_hf: -1
nfhout_hf: 3
nsout: -1
+output_fh: -1
diff --git a/parm/hafs.conf b/parm/hafs.conf
index 0784173b5..d8c2d17a0 100644
--- a/parm/hafs.conf
+++ b/parm/hafs.conf
@@ -190,9 +190,9 @@ lnsigcutoff=1.3 ;; length for vertical localization
[forecast]
# ccpp suites
-ccpp_suite_regional=HAFS_v0_gfdlmp_tedmf
-ccpp_suite_glob=HAFS_v0_gfdlmp_tedmf
-ccpp_suite_nest=HAFS_v0_gfdlmp_tedmf
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf
+ccpp_suite_glob=FV3_HAFS_v0_gfdlmp_tedmf
+ccpp_suite_nest=FV3_HAFS_v0_gfdlmp_tedmf
#radiation scheme calling time steps
fhswr=1800.
diff --git a/parm/hafs_globnest.conf b/parm/hafs_globnest.conf
index 23b719725..c9463b719 100644
--- a/parm/hafs_globnest.conf
+++ b/parm/hafs_globnest.conf
@@ -26,8 +26,8 @@ glob_layouty=10
glob_npx=769
glob_npy=769
# For the nested or regional standalone domain
-layoutx=40
-layouty=30
+layoutx=30
+layouty=40
npx=2561
npy=2161
npz=64
diff --git a/parm/hafs_globnest_static.conf b/parm/hafs_globnest_static.conf
index a9319c3be..63b7af0f1 100644
--- a/parm/hafs_globnest_static.conf
+++ b/parm/hafs_globnest_static.conf
@@ -56,8 +56,8 @@ glob_npy=769
# For the nested or regional standalone domain
#zzlayoutx=48
#zzlayouty=32
-layoutx=40
-layouty=30
+layoutx=30
+layouty=40
npx=2881
npy=1921
npz=64
diff --git a/parm/hafs_hycom.conf b/parm/hafs_hycom.conf
index 2c492fe25..a3aa0ff59 100644
--- a/parm/hafs_hycom.conf
+++ b/parm/hafs_hycom.conf
@@ -15,7 +15,7 @@ CASE_ROOT={config/case_root}
[forecast]
# Turn off nsst
-ccpp_suite_regional=HAFS_v0_gfdlmp_tedmf_nonsst
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf_nonsst
# Choose nstf_name=0,0,0,0,0
nstf_n1=0
nstf_n2=0
@@ -37,7 +37,7 @@ ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31
merge_import=.true.
[forecast_ens]
-ccpp_suite_regional_ens=HAFS_v0_gfdlmp_tedmf_nonsst
+ccpp_suite_regional_ens=FV3_HAFS_v0_gfdlmp_tedmf_nonsst
#
[rocotostr]
diff --git a/parm/hafs_regional_da_C96s1n4_320x312.conf b/parm/hafs_regional_da_C96s1n4_320x312.conf
index 18d30d992..21015b8d3 100644
--- a/parm/hafs_regional_da_C96s1n4_320x312.conf
+++ b/parm/hafs_regional_da_C96s1n4_320x312.conf
@@ -64,7 +64,7 @@ iend_nest_ens=175
jend_nest_ens=173
[forecast]
-ccpp_suite_regional=HAFS_v0_gfdlmp
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval="3 4 5 6 7 8 9" ;; restart interval in hours for the forecast
diff --git a/parm/hafs_regional_da_C96s1n4_dualres.conf b/parm/hafs_regional_da_C96s1n4_dualres.conf
index 96c19ee97..e38be2e27 100644
--- a/parm/hafs_regional_da_C96s1n4_dualres.conf
+++ b/parm/hafs_regional_da_C96s1n4_dualres.conf
@@ -64,7 +64,7 @@ iend_nest_ens=175
jend_nest_ens=173
[forecast]
-ccpp_suite_regional=HAFS_v0_gfdlmp
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval="3 6 9" ;; restart interval in hours for the forecast
@@ -86,7 +86,7 @@ output_grid_dlon=0.25 ;; output grid spacing dlon . . . .
output_grid_dlat=0.25 ;; output grid spacing dlat . . . .
[forecast_ens]
-ccpp_suite_regional_ens=HAFS_v0_gfdlmp
+ccpp_suite_regional_ens=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval_ens="6 12" ;; restart interval in hours for the ensda member forecast
diff --git a/parm/hafsv0p1aL64_da_AL.conf b/parm/hafsv0p1aL64_da_AL.conf
index a520cba51..e6d9cb8dc 100644
--- a/parm/hafsv0p1aL64_da_AL.conf
+++ b/parm/hafsv0p1aL64_da_AL.conf
@@ -60,7 +60,7 @@ iend_nest_ens=1485
jend_nest_ens=1367
[forecast]
-ccpp_suite_regional=HAFS_v0_gfdlmp
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval="3 6 9" ;; restart interval in hours for the forecast
@@ -79,7 +79,7 @@ output_grid_dlon=0.03 ;; output grid spacing dlon . . . .
output_grid_dlat=0.03 ;; output grid spacing dlat . . . .
[forecast_ens]
-ccpp_suite_regional_ens=HAFS_v0_gfdlmp
+ccpp_suite_regional_ens=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval_ens="6 12" ;; restart interval in hours for the ensda member forecast
diff --git a/parm/hafsv0p1aL91_da_AL.conf b/parm/hafsv0p1aL91_da_AL.conf
index 04cd25c2a..bd71a0243 100644
--- a/parm/hafsv0p1aL91_da_AL.conf
+++ b/parm/hafsv0p1aL91_da_AL.conf
@@ -61,7 +61,7 @@ iend_nest_ens=1485
jend_nest_ens=1367
[forecast]
-ccpp_suite_regional=HAFS_v0_gfdlmp
+ccpp_suite_regional=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval="3 6 9" ;; restart interval in hours for the forecast
@@ -80,7 +80,7 @@ output_grid_dlon=0.03 ;; output grid spacing dlon . . . .
output_grid_dlat=0.03 ;; output grid spacing dlat . . . .
[forecast_ens]
-ccpp_suite_regional_ens=HAFS_v0_gfdlmp
+ccpp_suite_regional_ens=FV3_HAFS_v0_gfdlmp_tedmf
restart_interval_ens="6 12" ;; restart interval in hours for the ensda member forecast
diff --git a/rocoto/sites/hera.ent b/rocoto/sites/hera.ent
index 6f38f7a06..180178e48 100644
--- a/rocoto/sites/hera.ent
+++ b/rocoto/sites/hera.ent
@@ -97,17 +97,27 @@
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
diff --git a/rocoto/sites/kjet.ent b/rocoto/sites/kjet.ent
index 64fa7c523..29fbc8df4 100644
--- a/rocoto/sites/kjet.ent
+++ b/rocoto/sites/kjet.ent
@@ -49,6 +49,9 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
@@ -85,20 +88,36 @@
79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;">
103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+ 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;">
+ 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;">
+
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
@@ -146,6 +165,7 @@
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;">
+
16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
@@ -155,6 +175,15 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;">
+ 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+
1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:0024G">
diff --git a/rocoto/sites/orion.ent b/rocoto/sites/orion.ent
index 0f76f2b7a..447a9e596 100644
--- a/rocoto/sites/orion.ent
+++ b/rocoto/sites/orion.ent
@@ -49,6 +49,9 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
@@ -85,20 +88,36 @@
79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;">
103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+ 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;">
+ 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;">
+
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
@@ -146,6 +165,7 @@
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;">
+
16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
@@ -155,6 +175,15 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;">
+ 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+
1:ppn=40:tpp=1TOTAL_TASKS40NCTSK40OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:0024G">
diff --git a/rocoto/sites/wcoss_cray.ent b/rocoto/sites/wcoss_cray.ent
index f7db0c4ff..8b7e6f057 100644
--- a/rocoto/sites/wcoss_cray.ent
+++ b/rocoto/sites/wcoss_cray.ent
@@ -90,16 +90,25 @@
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
diff --git a/rocoto/sites/wcoss_dell_p3.ent b/rocoto/sites/wcoss_dell_p3.ent
index 50224234e..17a3ba108 100644
--- a/rocoto/sites/wcoss_dell_p3.ent
+++ b/rocoto/sites/wcoss_dell_p3.ent
@@ -91,16 +91,25 @@
52:ppn=12:tpp=1TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=1TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=1TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=1TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=1TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=1TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=1TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=1TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=1TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=1TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=1TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=1TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=1TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=1TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=1TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=1TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=1TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=1TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=1TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=1TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=1TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=1TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=1TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
diff --git a/rocoto/sites/xjet.ent b/rocoto/sites/xjet.ent
index 8430ecbcf..da6ebdc42 100644
--- a/rocoto/sites/xjet.ent
+++ b/rocoto/sites/xjet.ent
@@ -50,6 +50,9 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
@@ -86,20 +89,36 @@
79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;">
103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+ 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;">
+ 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;">
+
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
@@ -147,6 +166,7 @@
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;">
+
16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
@@ -156,6 +176,15 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;">
+ 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+
1:ppn=12:tpp=1TOTAL_TASKS12NCTSK12OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:0024G">
diff --git a/rocoto/sites/xjet_hafsv0p2a.ent b/rocoto/sites/xjet_hafsv0p2a.ent
index 0f7e03fc3..b0a4544a1 100644
--- a/rocoto/sites/xjet_hafsv0p2a.ent
+++ b/rocoto/sites/xjet_hafsv0p2a.ent
@@ -50,6 +50,9 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
100:ppn=20:tpp=2TOTAL_TASKS2000NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
@@ -86,20 +89,36 @@
79:ppn=20:tpp=2TOTAL_TASKS1580NCTSK20&FORECAST_EXTRA;">
103:ppn=20:tpp=2TOTAL_TASKS2060NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+ 82:ppn=20:tpp=2TOTAL_TASKS1640NCTSK20&FORECAST_EXTRA;">
+ 106:ppn=20:tpp=2TOTAL_TASKS2120NCTSK20&FORECAST_EXTRA;">
+
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
52:ppn=12:tpp=2TOTAL_TASKS624NCTSK12&FORECAST_EXTRA;">
140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
+ 140:ppn=12:tpp=2TOTAL_TASKS1680NCTSK12&FORECAST_EXTRA;">
+ 148:ppn=12:tpp=2TOTAL_TASKS1776NCTSK12&FORECAST_EXTRA;">
53:ppn=12:tpp=2TOTAL_TASKS636NCTSK12&FORECAST_EXTRA;">
141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
+ 141:ppn=12:tpp=2TOTAL_TASKS1692NCTSK12&FORECAST_EXTRA;">
+ 149:ppn=12:tpp=2TOTAL_TASKS1788NCTSK12&FORECAST_EXTRA;">
54:ppn=12:tpp=2TOTAL_TASKS648NCTSK12&FORECAST_EXTRA;">
142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
+ 142:ppn=12:tpp=2TOTAL_TASKS1704NCTSK12&FORECAST_EXTRA;">
+ 150:ppn=12:tpp=2TOTAL_TASKS1800NCTSK12&FORECAST_EXTRA;">
88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 88:ppn=20:tpp=2TOTAL_TASKS1760NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
+ 91:ppn=20:tpp=2TOTAL_TASKS1820NCTSK20&FORECAST_EXTRA;">
6:ppn=12:tpp=2TOTAL_TASKS72NCTSK12&FORECAST_EXTRA;">
@@ -147,6 +166,7 @@
4:ppn=20:tpp=2TOTAL_TASKS80NCTSK20&FORECAST_EXTRA;">
7:ppn=20:tpp=2TOTAL_TASKS140NCTSK20&FORECAST_EXTRA;">
+
16:ppn=20:tpp=2TOTAL_TASKS320NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
@@ -156,6 +176,15 @@
64:ppn=20:tpp=2TOTAL_TASKS1280NCTSK20&FORECAST_EXTRA;">
67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 19:ppn=20:tpp=2TOTAL_TASKS380NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 22:ppn=20:tpp=2TOTAL_TASKS440NCTSK20&FORECAST_EXTRA;">
+ 25:ppn=20:tpp=2TOTAL_TASKS500NCTSK20&FORECAST_EXTRA;">
+ 31:ppn=20:tpp=2TOTAL_TASKS620NCTSK20&FORECAST_EXTRA;">
+ 34:ppn=20:tpp=2TOTAL_TASKS680NCTSK20&FORECAST_EXTRA;">
+ 67:ppn=20:tpp=2TOTAL_TASKS1340NCTSK20&FORECAST_EXTRA;">
+ 70:ppn=20:tpp=2TOTAL_TASKS1400NCTSK20&FORECAST_EXTRA;">
+
1:ppn=12:tpp=1TOTAL_TASKS12NCTSK12OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:00">
1:ppn=1:tpp=1TOTAL_TASKS1NCTSK1OMP_THREADS107:59:0024G">
diff --git a/scripts/exhafs_forecast.sh b/scripts/exhafs_forecast.sh
index cdfb7227c..33b8ccbef 100755
--- a/scripts/exhafs_forecast.sh
+++ b/scripts/exhafs_forecast.sh
@@ -42,9 +42,9 @@ if [ "${ENSDA}" != YES ]; then
target_lat=${target_lat:-22.0}
refine_ratio=${refine_ratio:-4}
deflate_level=${deflate_level:--1}
- ccpp_suite_regional=${ccpp_suite_regional:-HAFS_v0_gfdlmp}
- ccpp_suite_glob=${ccpp_suite_glob:-HAFS_v0_gfdlmp}
- ccpp_suite_nest=${ccpp_suite_nest:-HAFS_v0_gfdlmp}
+ ccpp_suite_regional=${ccpp_suite_regional:-FV3_HAFS_v1}
+ ccpp_suite_glob=${ccpp_suite_glob:-FV3_HAFS_v1}
+ ccpp_suite_nest=${ccpp_suite_nest:-FV3_HAFS_v1}
dt_atmos=${dt_atmos:-90}
restart_interval=${restart_interval:-6}
quilting=${quilting:-.true.}
@@ -79,9 +79,9 @@ else
target_lat=${target_lat_ens:-22.0}
refine_ratio=${refine_ratio_ens:-4}
deflate_level=${deflate_level:-1}
- ccpp_suite_regional=${ccpp_suite_regional_ens:-HAFS_v0_gfdlmp}
- ccpp_suite_glob=${ccpp_suite_glob_ens:-HAFS_v0_gfdlmp}
- ccpp_suite_nest=${ccpp_suite_nest_ens:-HAFS_v0_gfdlmp}
+ ccpp_suite_regional=${ccpp_suite_regional_ens:-FV3_HAFS_v1}
+ ccpp_suite_glob=${ccpp_suite_glob_ens:-FV3_HAFS_v1}
+ ccpp_suite_nest=${ccpp_suite_nest_ens:-FV3_HAFS_v1}
dt_atmos=${dt_atmos_ens:-90}
restart_interval=${restart_interval_ens:-6}
quilting=${quilting_ens:-.true.}
@@ -355,7 +355,7 @@ cd ..
# Prepare data_table, diag_table, field_table, input.nml, input_nest02.nml,
# model_configure, and nems.configure
-${NCP} ${PARMforecast}/data_table .
+#${NCP} ${PARMforecast}/data_table .
${NCP} ${PARMforecast}/diag_table.tmp .
${NCP} ${PARMforecast}/field_table .
${NCP} ${PARMforecast}/input.nml.tmp .
@@ -369,8 +369,7 @@ ioffset=$(( (istart_nest-1)/2 + 1))
joffset=$(( (jstart_nest-1)/2 + 1))
blocksize=$(( ${glob_npy}/${glob_layouty} ))
-sed -e "s/_fhmax_/${NHRS}/g" \
- -e "s/_blocksize_/${blocksize:-64}/g" \
+sed -e "s/_blocksize_/${blocksize:-64}/g" \
-e "s/_ccpp_suite_/${ccpp_suite_glob}/g" \
-e "s/_deflate_level_/${deflate_level:--1}/g" \
-e "s/_layoutx_/${glob_layoutx}/g" \
@@ -406,8 +405,7 @@ sed -e "s/_fhmax_/${NHRS}/g" \
input.nml.tmp > input.nml
blocksize=$(( ${npy}/${layouty} ))
-sed -e "s/_fhmax_/${NHRS}/g" \
- -e "s/_blocksize_/${blocksize:-64}/g" \
+sed -e "s/_blocksize_/${blocksize:-64}/g" \
-e "s/_ccpp_suite_/${ccpp_suite_nest}/g" \
-e "s/_deflate_level_/${deflate_level:--1}/g" \
-e "s/_layoutx_/${layoutx}/g" \
@@ -474,7 +472,7 @@ cd ..
# Prepare data_table, diag_table, field_table, input.nml, input_nest02.nml,
# model_configure, and nems.configure
-${NCP} ${PARMforecast}/data_table .
+#${NCP} ${PARMforecast}/data_table .
${NCP} ${PARMforecast}/diag_table.tmp .
${NCP} ${PARMforecast}/field_table .
${NCP} ${PARMforecast}/input.nml.tmp .
@@ -504,8 +502,7 @@ else
fi
blocksize=$(( ${npy}/${layouty} ))
-sed -e "s/_fhmax_/${NHRS}/g" \
- -e "s/_blocksize_/${blocksize:-64}/g" \
+sed -e "s/_blocksize_/${blocksize:-64}/g" \
-e "s/_ccpp_suite_/${ccpp_suite_regional}/g" \
-e "s/_deflate_level_/${deflate_level:--1}/g" \
-e "s/_layoutx_/${layoutx}/g" \
@@ -612,10 +609,8 @@ $yr $mn $dy $cyc 0 0
EOF
cat temp diag_table.tmp > diag_table
-sed -e "s/NTASKS/${TOTAL_TASKS}/g" -e "s/YR/$yr/g" \
- -e "s/MN/$mn/g" -e "s/DY/$dy/g" -e "s/H_R/$cyc/g" \
- -e "s/NHRS/$NHRS/g" -e "s/NTHRD/$OMP_NUM_THREADS/g" \
- -e "s/NCNODE/$NCNODE/g" \
+sed -e "s/YR/$yr/g" -e "s/MN/$mn/g" -e "s/DY/$dy/g" \
+ -e "s/H_R/$cyc/g" -e "s/NHRS/$NHRS/g" \
-e "s/_dt_atmos_/${dt_atmos}/g" \
-e "s/_restart_interval_/${restart_interval}/g" \
-e "s/_quilting_/${quilting}/g" \
@@ -633,7 +628,6 @@ sed -e "s/NTASKS/${TOTAL_TASKS}/g" -e "s/YR/$yr/g" \
-e "s/_LAT2_/$output_grid_lat2/g" \
-e "s/_DLON_/$output_grid_dlon/g" \
-e "s/_DLAT_/$output_grid_dlat/g" \
- -e "s/_cpl_/${cplflx:-.false.}/g" \
model_configure.tmp > model_configure
# Copy fix files needed by inline_post
diff --git a/scripts/exhafs_hrdgraphics.sh b/scripts/exhafs_hrdgraphics.sh
index 64c810fac..73a48dc1c 100755
--- a/scripts/exhafs_hrdgraphics.sh
+++ b/scripts/exhafs_hrdgraphics.sh
@@ -14,6 +14,7 @@ export MP_LABELIO=yes
CDATE=${CDATE:-${YMDH}}
NHRS=${NHRS:-126}
+NHR3=$( printf "%03d" "$NHRS" )
NOUTHRS=${NOUTHRS:-3}
machine=${machine:-jet}
@@ -41,7 +42,7 @@ BDECKhafs=${BDECKhafs:-/lfs1/HFIP/hur-aoml/Ghassan.Alaka/bdeck}
SYNDAThafs=${SYNDAThafs:-/lfs4/HFIP/hwrf-data/hwrf-input/SYNDAT-PLUS}
# Setup the working directory and change into it
-COMgplot=${COMgplot:-${COMhafs}/hrdgraphics/}
+COMgplot=${COMgplot:-${COMhafs}/hrdgraphics}
WORKgplot=${WORKgplot:-${WORKhafs}/hrdgraphics}
mkdir -p ${COMgplot}
mkdir -p ${WORKgplot}
@@ -51,7 +52,12 @@ cd ${WORKgplot}
NML=${WORKgplot}/namelist.master.${SUBEXPT}
if [ ! -f ${NML} ];
then
- cp -p ${GPLOThafs}/nmlist/namelist.master.HAFS_Default ${NML}
+ if [ -f ${GPLOThafs}/nmlist/namelist.master.${SUBEXPT} ];
+ then
+ cp -p ${GPLOThafs}/nmlist/namelist.master.${SUBEXPT} ${NML}
+ else
+ cp -p ${GPLOThafs}/nmlist/namelist.master.HAFS_Default ${NML}
+ fi
fi
sed -i 's/^EXPT =.*/EXPT = '"${SUBEXPT}"'/g' ${NML}
sed -i 's/^IDATE =.*/IDATE = '"${CDATE}"'/g' ${NML}
@@ -61,7 +67,7 @@ sed -i 's@^IDIR =.*@IDIR = '"${COMhafs}"'@g' ${NML}
sed -i 's@^ODIR =.*@ODIR = '"${WORKgplot}"'@g' ${NML}
sed -i 's@^ATCF1_DIR =.*@ATCF1_DIR = '"${COMhafs}"'@g' ${NML}
sed -i 's@^ATCF2_DIR =.*@ATCF2_DIR = '"${CDNOSCRUB}/${SUBEXPT}"'@g' ${NML}
-sed -i 's@^ADECK_DIR =.*@ADECK_DIR = '"${ADECKhafs}/${SUBEXPT}"'@g' ${NML}
+sed -i 's@^ADECK_DIR =.*@ADECK_DIR = '"${ADECKhafs}"'@g' ${NML}
sed -i 's@^BDECK_DIR =.*@BDECK_DIR = '"${BDECKhafs}"'@g' ${NML}
sed -i 's/^SYS_ENV =.*/SYS_ENV = '"$( echo ${machine} | tr "[a-z]" "[A-Z]")"'/g' ${NML}
sed -i 's/^BATCH_MODE =.*/BATCH_MODE = Background/g' ${NML}
@@ -69,7 +75,7 @@ sed -i 's/^BATCH_MODE =.*/BATCH_MODE = Background/g' ${NML}
# Initialize ALL_COMPLETE as false
ALL_COMPLETE=0
-# Loop for forecast hours
+# Loop until everything is complete (ALL_COMPLETE=1)
while [[ ${ALL_COMPLETE} -eq 0 ]];
do
echo "Top of loop"
@@ -79,7 +85,8 @@ do
${GPLOT_PARSE} HAFS ${COMhafs} ${COMhafs} ${BDECKhafs} ${SYNDAThafs} 4
${GPLOT_PARSE} HAFS ${CDNOSCRUB}/${SUBEXPT} ${CDNOSCRUB}/${SUBEXPT} ${BDECKhafs} ${SYNDAThafs} 0 "*${DATE}*.atcfunix.all"
- # Check the status files for all GPLOT components.
+ # Check the status logs for all GPLOT components.
+ # If every log doesn't say "complete", set ALL_COMPLETE=0
GPLOT_STATUS=( `find ${WORKgplot} -name "status.*" -exec cat {} \;` )
ALL_COMPLETE=1
if [ ! -z "${GPLOT_STATUS[*]}" ];
@@ -94,20 +101,28 @@ do
ALL_COMPLETE=0
fi
+ # Check that the final HAFS output has been post-processed by atm_post.
+ # If not, set ALL_COMPLETE=0
+ if [ ! -f ${WORKhafs}/forecast/postf${NHR3} ];
+ then
+ ALL_COMPLETE=0
+ echo "This file doesn't exist --> ${WORKhafs}/forecast/postf${NHR3}"
+ echo "That means the final HAFS output has not been post-processed by atm_post."
+ fi
+
# Deliver all new and modified graphics to COMhafs/graphics
- # Note: a fatal error (24) occurs when a file staged for transfer vanishes.
- #cp -rup ${WORKgplot} ${COMgplot}
- ${USHhafs}/rsync-no-vanished.sh -av --include="*/" --include="*gif" --exclude="*" ${WORKgplot}/. ${COMgplot}/.
+ ${USHhafs}/rsync-no-vanished.sh -av --include="*/" --include="*gif" --include="*dat" --exclude="*" ${WORKgplot}/. ${COMgplot}/.
rsync -av --include="*.atcfunix*" --exclude="*" ${COMhafs}/. ${COMgplot}/.
- # If all are complete, then exit with success!
+ # If all status logs are complete and the final output has been processed
+ # by atm_post, then exit with success!
# If not, submit the GPLOT wrapper again.
if [[ ${ALL_COMPLETE} -eq 1 ]];
then
- echo "All status files were found to read 'complete.' That means we're done!"
+ echo "All status logs say 'complete' and the final output has been processed by atm_post (fhr=${NHRS}). That means we're done!"
break
else
- echo "Waiting for all status files to read 'complete'. Trying again..."
+ echo "Waiting for all status logs to say 'complete' and for atm_post to process the final output (fhr=${NHRS}). Trying again..."
fi
# Call the GPLOT wrapper
@@ -122,7 +137,7 @@ done
# Now that everything is complete, move all graphics to the $COMhafs directory.
if [ "${SENDCOM}" == "YES" ]; then
#cp -rup ${WORKgplot} ${COMgplot}
- ${USHhafs}/rsync-no-vanished.sh -av --include="*/" --include="*gif" --exclude="*" ${WORKgplot}/. ${COMgplot}/.
+ ${USHhafs}/rsync-no-vanished.sh -av --include="*/" --include="*gif" --include="*dat" --exclude="*" ${WORKgplot}/. ${COMgplot}/.
rsync -av --include="*.atcfunix*" --exclude="*" ${COMhafs}/. ${COMgplot}/.
fi
diff --git a/sorc/build_forecast.sh b/sorc/build_forecast.sh
index ebb5ca40e..301cbd203 100755
--- a/sorc/build_forecast.sh
+++ b/sorc/build_forecast.sh
@@ -9,6 +9,6 @@ if [ $target = jet ]; then target=jet.intel ; fi
cd hafs_forecast.fd/tests
-./compile.sh "$target" "-DAPP=HAFS -DCCPP_SUITES=HAFS_v0_gfdlmp_tedmf_nonsst,HAFS_v0_gfdlmp_tedmf,HAFS_v0_gfdlmp_nocpnsst,HAFS_v0_gfdlmp_nonsst,HAFS_v0_gfdlmp_nocp,HAFS_v0_gfdlmp,HAFS_v0_hwrf_thompson,HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO
+./compile.sh "$target" "-DAPP=HAFS -DCCPP_SUITES=FV3_HAFS_v0_gfdlmp_tedmf_nonsst,FV3_HAFS_v0_gfdlmp_tedmf,FV3_HAFS_v0_hwrf_thompson,FV3_HAFS_v0_hwrf -D32BIT=ON" 32bit YES NO
exit
diff --git a/sorc/build_hycom_utils.sh b/sorc/build_hycom_utils.sh
index 2e67ed291..5b073fb75 100755
--- a/sorc/build_hycom_utils.sh
+++ b/sorc/build_hycom_utils.sh
@@ -20,10 +20,13 @@ fi
mkdir build
cd build
if [ $target = wcoss_cray ]; then
- cmake .. -DCMAKE_Fortran_COMPILER=ftn -DCMAKE_C_COMPILER=cc
+ CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-ftn}
+ CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-cc}
else
- cmake .. -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_C_COMPILER=icc
+ CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-ifort}
+ CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-icc}
fi
+cmake .. -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
make -j 8 VERBOSE=1
cd ${cwd}/hafs_hycom_utils.fd
@@ -33,10 +36,13 @@ fi
mkdir build
cd build
if [ $target = wcoss_cray ]; then
- cmake .. -DCMAKE_Fortran_COMPILER=ftn -DCMAKE_C_COMPILER=cc
+ CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-ftn}
+ CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-cc}
else
- cmake .. -DCMAKE_Fortran_COMPILER=ifort -DCMAKE_C_COMPILER=icc
+ CMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER:-ifort}
+ CMAKE_C_COMPILER=${CMAKE_C_COMPILER:-icc}
fi
+cmake .. -DCMAKE_Fortran_COMPILER=${CMAKE_Fortran_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
make -j 8 VERBOSE=1
make install
diff --git a/sorc/hafs_forecast.fd b/sorc/hafs_forecast.fd
index e5cb32e9f..c1e2e1d95 160000
--- a/sorc/hafs_forecast.fd
+++ b/sorc/hafs_forecast.fd
@@ -1 +1 @@
-Subproject commit e5cb32e9ff65c4b434879733aef79ab72157b3d7
+Subproject commit c1e2e1d95236df6be1d738c478b3513a35087093
diff --git a/sorc/hafs_graphics.fd/emc_graphics b/sorc/hafs_graphics.fd/emc_graphics
index 92b6b535d..db483c53c 160000
--- a/sorc/hafs_graphics.fd/emc_graphics
+++ b/sorc/hafs_graphics.fd/emc_graphics
@@ -1 +1 @@
-Subproject commit 92b6b535d728d7ea97ec5e39659515f2fe6e88d0
+Subproject commit db483c53cdefddc64397f6bb49f486be5844fb45
diff --git a/sorc/hafs_graphics.fd/hrd_gplot b/sorc/hafs_graphics.fd/hrd_gplot
index 48d6a7045..b12afbb2b 160000
--- a/sorc/hafs_graphics.fd/hrd_gplot
+++ b/sorc/hafs_graphics.fd/hrd_gplot
@@ -1 +1 @@
-Subproject commit 48d6a704503a9ad8ba059a69991b50994eae3f79
+Subproject commit b12afbb2bac0a19c1e77bc8779fea71569335d30
diff --git a/sorc/hafs_gsi.fd b/sorc/hafs_gsi.fd
index 2b182d5ad..af62451c1 160000
--- a/sorc/hafs_gsi.fd
+++ b/sorc/hafs_gsi.fd
@@ -1 +1 @@
-Subproject commit 2b182d5ad603a95b067cafd947772887f7e4375b
+Subproject commit af62451c139f608b9983a0d826d1f243ef1920c1
diff --git a/sorc/hafs_post.fd b/sorc/hafs_post.fd
index 3c9ff6514..fc819611b 160000
--- a/sorc/hafs_post.fd
+++ b/sorc/hafs_post.fd
@@ -1 +1 @@
-Subproject commit 3c9ff6514648b224a3758a9e17f4ab59c2e56de5
+Subproject commit fc819611b81001feaee37f776847ff90d2033d4b
diff --git a/sorc/hafs_utils.fd b/sorc/hafs_utils.fd
index e2a147914..94cfb6c22 160000
--- a/sorc/hafs_utils.fd
+++ b/sorc/hafs_utils.fd
@@ -1 +1 @@
-Subproject commit e2a147914ccf69ac2b2f9e77e2b5ed233cf08580
+Subproject commit 94cfb6c22f8dff9e72d0d1b399f814514e0008ca