From c1961769596b1dfd2c7f72ef82b7df79ce07cc69 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 7 Jun 2021 18:43:46 +0000
Subject: [PATCH 1/6] Improve cycle cadence specification in CROW

Replaces the old gfs_cyc method for determining cycling frequency with one that
explicitly sets the step length. In place of gfs_cyc there are two new variables
to determine step sizing: STEP_GFS and STEP_DA. Both are of the timedelta type.

STEP_GFS determines the frequency of GFS and defaults to 6 hours
STEP_DA determines the frequency of DA and also defaults to 6 hours

If STEP_GFS is set to 00:00:00, only the DA system is run
If STEP_DA is set to 00:00:00, only the GFS system is run (free-forecast mode)

Note that this new method currently requires the additional setting of STEP_DA
to zero, in addition to using free forecast layout.

If STEP_DA is non-zero, the first cycle is DA-only, so this should be kept in
mind when setting up your experiments (especially if you want GFS to run at
certain times of day). If both are non-zero, STEP_GFS should always be an
integer multiple of STEP_DA, otherwise unexpected behavior may occur. Other
than setting STEP_DA to zero for free forecast, STEP_DA is unlikely to be
changed from its default value in the near-future for GFS, but the new method
is flexible for any future incorporation of LAM system that requires faster DA
cycling.

Examples:

Full-cycling with DA & GFS:
STEP_GFS: !timedelta "06:00:00"
STEP_DA: !timedelta "06:00:00"
(since these are defaults, nothing needs to be placed in the case file, but you may
wish to for clarity)

Normal DA with GFS at each 00z starting 2013-04-01:
SDATE: 2013-03-31t18:00:00
STEP_GFS: !timedelta "24:00:00"

Free forecast with GFS every 24h at 00z starting 2013-04-01:
SDATE: 2013-04-01t00:00:00
STEP_GFS: !timedelta "24:00:00"
STEP_DA: !timedelta "00:00:00"

CROW ss updated to add a new tool that converts a timedelta to a string.

The wave model, which formerly used gfs_cyc to determine the wave cycling interval,
now determines WAVHCYC directly from STEP_GFS. Since this variable is only a number
of hours, this will only work for whole-hour GFS frequencies.

Case files have been updated to use the new settings in place of gfs_cyc, as well as
set STEP_DA to zero for free forecast cases. However, they have not all been tested.

Additional testing is needed to ensure these changes are fully operational for DA
cycling runs.

Refs: #260, #314
---
 workflow/CROW                                 |  2 +-
 workflow/cases/aerosol_firex_forecast.yaml    |  4 +-
 workflow/cases/aerosol_free_forecast.yaml     | 20 +++++---
 workflow/cases/atm_free_forecast.yaml         |  2 +
 workflow/cases/coupled_free_forecast.yaml     |  2 +
 .../coupled_free_forecast_nofrac_wave.yaml    |  2 +
 .../cases/coupled_free_forecast_wave.yaml     |  2 +
 workflow/cases/dev/Run-From-HPSS-ICs.yaml     |  5 +-
 ...prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml |  2 +-
 ...prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml |  2 +-
 ...19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml |  2 +-
 ...3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml |  2 +-
 ...9retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml |  2 +-
 ...tro5-GFS@C768_ENKF@384+80MEM_whatever.yaml |  2 +-
 .../fv3q2fy19retro5-GFS@C768_ff_fixed.yaml    |  3 +-
 workflow/cases/dev/gaea-test.yaml             |  2 +-
 workflow/cases/dev/lowrsecf@C384.yaml         |  2 +-
 workflow/cases/dev/merge2dell_baseline.yaml   |  2 +-
 workflow/cases/dev/merge2jet_baseline.yaml    |  2 +-
 workflow/cases/dev/nco.yaml                   |  2 +-
 .../dev/parallel@768_ENKF@384+80MEM.yaml      |  2 +-
 ...prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml |  2 +-
 ...time-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml |  2 +-
 workflow/cases/free_forecast_case.yaml        |  3 +-
 workflow/cases/jet_cycled.yaml                |  2 +-
 workflow/cases/jet_free_forecast.yaml         |  3 +-
 workflow/cases/manual_mode.yaml               |  2 +-
 workflow/cases/prototype6.yaml                |  2 +
 .../AtmRiv_2016021000_GFS@C192.yaml           |  2 +-
 .../AtmRiv_2016021000_GFS@C384.yaml           |  2 +-
 .../Harvey_2017081712_GFS@C192.yaml           |  2 +-
 ..._GFS@C384-override-stuff-in-case-file.yaml |  2 +-
 .../Harvey_2017081712_GFS@C384.yaml           |  2 +-
 .../Harvey_2017081712_GFS@C768.yaml           |  2 +-
 .../March_Noreasters_2018022800_GFS@C192.yaml |  2 +-
 .../March_Noreasters_2018022800_GFS@C384.yaml |  2 +-
 .../March_Noreasters_2018022800_GFS@C768.yaml |  2 +-
 workflow/cases/tutorial_cycled.yaml           |  2 +-
 workflow/cases/tutorial_free_forecast.yaml    |  3 +-
 workflow/config/base.yaml                     |  5 +-
 workflow/config/wave.yaml                     |  6 +--
 workflow/defaults/settings.yaml               |  4 +-
 workflow/layout/cycled_gfs.yaml               |  4 +-
 workflow/layout/free_forecast_gfs.yaml        |  5 +-
 workflow/layout/manual_temp.yaml              |  4 +-
 workflow/layout/nco.yaml                      |  2 +-
 workflow/runtime/suite.yaml                   | 47 ++++++++++++-------
 workflow/schema/settings.yaml                 | 23 ++++-----
 48 files changed, 117 insertions(+), 88 deletions(-)

diff --git a/workflow/CROW b/workflow/CROW
index 6065439cc6..2079a68118 160000
--- a/workflow/CROW
+++ b/workflow/CROW
@@ -1 +1 @@
-Subproject commit 6065439cc6bbffa61d5551dd8215877e3322d3e0
+Subproject commit 2079a681188ea49ad73f548de093216c828e8750
diff --git a/workflow/cases/aerosol_firex_forecast.yaml b/workflow/cases/aerosol_firex_forecast.yaml
index 4a881d2592..ee8e96b8c7 100644
--- a/workflow/cases/aerosol_firex_forecast.yaml
+++ b/workflow/cases/aerosol_firex_forecast.yaml
@@ -11,10 +11,10 @@ case:
   settings: 
     SDATE: 2019-07-01t00:00:00
     EDATE: 2019-07-15t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
     max_job_tries: 1
 
-    gfs_cyc: 1
-
     cplgocart: .true.
     print_esmf: .true.
     nems_temp: 'atm_aer'
diff --git a/workflow/cases/aerosol_free_forecast.yaml b/workflow/cases/aerosol_free_forecast.yaml
index 28923dd2c7..0c13067cfb 100644
--- a/workflow/cases/aerosol_free_forecast.yaml
+++ b/workflow/cases/aerosol_free_forecast.yaml
@@ -4,8 +4,11 @@ case:
 
   settings: 
     SDATE: 2013-04-01t00:00:00
-    EDATE: 2013-04-01t00:00:00
+    EDATE: 2013-04-02t00:00:00    
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
     max_job_tries: 1
+    rocoto_cycle_throttle: 1
 
     cplgocart: .true.
     print_esmf: .true.
@@ -35,12 +38,12 @@ case:
     LEVS: 65
     DELTIM: 450
     layout:
-       x: 6
-       y: 8
-       nth: 1
-       WGRP: 1
-       WGRP_NTASKS: 24
-       WRTIOBUF: "32M"
+      x: 6
+      y: 8
+      nth: 1
+      WGRP: 1
+      WGRP_NTASKS: 24
+      WRTIOBUF: "32M"
 
   chem_settings:
     MOD: gsdchem
@@ -55,3 +58,6 @@ case:
   post:
     downset: 2
     GOESF: no
+
+  archiving:
+    archive_to_hpss: False
diff --git a/workflow/cases/atm_free_forecast.yaml b/workflow/cases/atm_free_forecast.yaml
index 600e069044..719b6bda0f 100644
--- a/workflow/cases/atm_free_forecast.yaml
+++ b/workflow/cases/atm_free_forecast.yaml
@@ -6,6 +6,8 @@ case:
   settings: 
     SDATE: 2013-04-01t00:00:00
     EDATE: 2013-04-01t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     cplflx: .false.
     print_esmf: .true.
diff --git a/workflow/cases/coupled_free_forecast.yaml b/workflow/cases/coupled_free_forecast.yaml
index 2e31e51a14..f7c6e0e39f 100644
--- a/workflow/cases/coupled_free_forecast.yaml
+++ b/workflow/cases/coupled_free_forecast.yaml
@@ -5,6 +5,8 @@ case:
   settings: 
     SDATE: 2013-04-01t00:00:00
     EDATE: 2013-04-01t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     onestep: .true.
     cplmode: nems_frac
diff --git a/workflow/cases/coupled_free_forecast_nofrac_wave.yaml b/workflow/cases/coupled_free_forecast_nofrac_wave.yaml
index 161708b483..fca4453777 100644
--- a/workflow/cases/coupled_free_forecast_nofrac_wave.yaml
+++ b/workflow/cases/coupled_free_forecast_nofrac_wave.yaml
@@ -5,6 +5,8 @@ case:
   settings: 
     SDATE: 2013-04-01t00:00:00
     EDATE: 2013-04-01t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     onestep: .true.
     cplmode: nems_orig
diff --git a/workflow/cases/coupled_free_forecast_wave.yaml b/workflow/cases/coupled_free_forecast_wave.yaml
index 72b96eee1e..acf39609da 100644
--- a/workflow/cases/coupled_free_forecast_wave.yaml
+++ b/workflow/cases/coupled_free_forecast_wave.yaml
@@ -5,6 +5,8 @@ case:
   settings: 
     SDATE: 2013-04-01t00:00:00
     EDATE: 2013-04-01t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     onestep: .true.
     cplmode: nems_frac
diff --git a/workflow/cases/dev/Run-From-HPSS-ICs.yaml b/workflow/cases/dev/Run-From-HPSS-ICs.yaml
index 9627d778b6..88cb6d0bc6 100644
--- a/workflow/cases/dev/Run-From-HPSS-ICs.yaml
+++ b/workflow/cases/dev/Run-From-HPSS-ICs.yaml
@@ -57,10 +57,7 @@ case:
     # files instead.
     chgres_and_convert_ics: yes
 
-    # gfs_cyc: this setting is used in a full data assimilation cycled
-    # workflow to only run the GFS every N cycles.  Keep this at "4" to
-    # ensure it is run every cycle.  
-    gfs_cyc: 4
+    STEP_GFS: !timedelta "06:00:00"
 
   places:
     # workflow_file: which workflow we're running.  This specifies the
diff --git a/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
index 0541c3cf45..a46525e4c6 100644
--- a/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
+++ b/workflow/cases/dev/ecrealtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
@@ -136,5 +136,5 @@ case:
     EDATE: 2018-08-30t18:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
index d25582577d..0b010d8c63 100644
--- a/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
+++ b/workflow/cases/dev/four_cycle_mode_prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
@@ -141,7 +141,7 @@ case:
     EDATE: 2018-08-10t18:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
 
   schedvar:
diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml
index a3f89871f3..b52f5a67fc 100644
--- a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml
+++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM-Dell.yaml
@@ -134,5 +134,5 @@ case:
     EDATE: 2016-04-18t12:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml
index 3ba368393e..46a5a72e9c 100644
--- a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml
+++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM.yaml
@@ -172,5 +172,5 @@ case:
     EDATE: 2016-05-31t18:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml
index 7783972108..5110e6d35d 100644
--- a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml
+++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_fixed.yaml
@@ -186,5 +186,5 @@ case:
     EDATE: 2016-02-14t12:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml
index 32964f0caf..baba21188e 100644
--- a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml
+++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ENKF@384+80MEM_whatever.yaml
@@ -186,5 +186,5 @@ case:
     EDATE: 2016-02-14t12:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml
index 59a6780d4e..ff649c4940 100644
--- a/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml
+++ b/workflow/cases/dev/fv3q2fy19retro5-GFS@C768_ff_fixed.yaml
@@ -95,5 +95,6 @@ case:
     EDATE: 2016-02-14t12:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
+    STEP_DA: !timedelta "00:00:00"
     run_gsi: No
diff --git a/workflow/cases/dev/gaea-test.yaml b/workflow/cases/dev/gaea-test.yaml
index f59cb6275f..323f51472a 100644
--- a/workflow/cases/dev/gaea-test.yaml
+++ b/workflow/cases/dev/gaea-test.yaml
@@ -21,5 +21,5 @@ case:
     SDATE: 2015-11-26t00:00:00
     EDATE: 2015-11-26t00:00:00
     realtime: false
-    gfs_cyc: 4 # run gfs four cycles a day
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: No # disable data assimilation
diff --git a/workflow/cases/dev/lowrsecf@C384.yaml b/workflow/cases/dev/lowrsecf@C384.yaml
index 4ee8f3697f..7beb035b03 100644
--- a/workflow/cases/dev/lowrsecf@C384.yaml
+++ b/workflow/cases/dev/lowrsecf@C384.yaml
@@ -31,7 +31,7 @@ case:
     EDATE: 2018-05-31t12:00:00
     chgres_and_convert_ics: No
     REALTIME: true
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: YES
 
   post:
diff --git a/workflow/cases/dev/merge2dell_baseline.yaml b/workflow/cases/dev/merge2dell_baseline.yaml
index 39641f897e..0a82133dbe 100644
--- a/workflow/cases/dev/merge2dell_baseline.yaml
+++ b/workflow/cases/dev/merge2dell_baseline.yaml
@@ -139,7 +139,7 @@ case:
     EDATE: 2018-11-26t12:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
 
   schedvar:
diff --git a/workflow/cases/dev/merge2jet_baseline.yaml b/workflow/cases/dev/merge2jet_baseline.yaml
index 4a4ba71afa..1004738f32 100644
--- a/workflow/cases/dev/merge2jet_baseline.yaml
+++ b/workflow/cases/dev/merge2jet_baseline.yaml
@@ -181,5 +181,5 @@ case:
     EDATE: 2019-05-02t00:00:00
     chgres_and_convert_ics: No
     realtime: false
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/nco.yaml b/workflow/cases/dev/nco.yaml
index 9d1d855acc..8fcb0195fd 100644
--- a/workflow/cases/dev/nco.yaml
+++ b/workflow/cases/dev/nco.yaml
@@ -136,7 +136,7 @@ case:
     EDATE: 2018-09-22t06:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
 
   schedvar:
diff --git a/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml b/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml
index e3eba3ceeb..25e0a76489 100644
--- a/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml
+++ b/workflow/cases/dev/parallel@768_ENKF@384+80MEM.yaml
@@ -34,7 +34,7 @@ case:
     IC_CDUMP: gdas # Get initial conditions from 1st cycle from GFS or GDAS
     SDATE: 2018-05-03t00:00:00
     EDATE: 2018-05-03t18:00:00
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
     chgres_and_convert_ics: No
 
diff --git a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
index 0a170b5df7..6363565a38 100644
--- a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
+++ b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM-Dell.yaml
@@ -119,5 +119,5 @@ case:
     EDATE: 2018-09-09t18:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
diff --git a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml
index b10dceb0ef..b304ba3056 100644
--- a/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml
+++ b/workflow/cases/dev/realtime-prfv3rt1_GFS@C768_ENKF@384+80MEM.yaml
@@ -42,6 +42,6 @@ case:
     EDATE: 2018-06-04t12:00:00
     chgres_and_convert_ics: No
     realtime: true
-    gfs_cyc: 4 # run gfs at 0z, 12z
+    STEP_GFS: !timedelta "06:00:00"
     run_gsi: Yes
 
diff --git a/workflow/cases/free_forecast_case.yaml b/workflow/cases/free_forecast_case.yaml
index 5f4f24c5bb..1e371bcbcb 100644
--- a/workflow/cases/free_forecast_case.yaml
+++ b/workflow/cases/free_forecast_case.yaml
@@ -9,10 +9,11 @@ case:
   settings: 
     SDATE: 2019-05-01t00:00:00
     EDATE: 2019-05-01t06:00:00
+    STEP_GFS: !timedelta "06:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     DUMP_SUFFIX: "p"
     run_gsi: No
     chgres_and_convert_ics: Yes
-    gfs_cyc: 4 # run gfs every cycle
 
     mom6ic_prepared: .true.
diff --git a/workflow/cases/jet_cycled.yaml b/workflow/cases/jet_cycled.yaml
index c00e9aefd4..c2a9e375c8 100644
--- a/workflow/cases/jet_cycled.yaml
+++ b/workflow/cases/jet_cycled.yaml
@@ -16,4 +16,4 @@ case:
     DUMP_SUFFIX: ""
     run_gsi: Yes
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/jet_free_forecast.yaml b/workflow/cases/jet_free_forecast.yaml
index 80fb80fc8e..7751d03f0d 100644
--- a/workflow/cases/jet_free_forecast.yaml
+++ b/workflow/cases/jet_free_forecast.yaml
@@ -14,11 +14,12 @@ case:
   settings: 
     SDATE: 2019-05-01t00:00:00
     EDATE: 2019-05-01t00:00:00
+    STEP_GFS: !timedelta "06:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     DUMP_SUFFIX: "p"
     run_gsi: No
     chgres_and_convert_ics: Yes
-    gfs_cyc: 4 # run gfs every cycle
 
   suite_overrides: !FirstTrue
     - when: !calc (doc.platform.name=="JET")
diff --git a/workflow/cases/manual_mode.yaml b/workflow/cases/manual_mode.yaml
index eb216fefd2..e39284265f 100644
--- a/workflow/cases/manual_mode.yaml
+++ b/workflow/cases/manual_mode.yaml
@@ -92,7 +92,7 @@ case:
     DUMP_SUFFIX: "p"
     run_gsi: Yes
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
 
   downstream: &downstream
     DO_POST_PROCESSING: NO
diff --git a/workflow/cases/prototype6.yaml b/workflow/cases/prototype6.yaml
index 3b6c7a639f..922503d661 100644
--- a/workflow/cases/prototype6.yaml
+++ b/workflow/cases/prototype6.yaml
@@ -5,6 +5,8 @@ case:
   settings: 
     SDATE: 2013-04-01t00:00:00
     EDATE: 2013-04-01t00:00:00
+    STEP_GFS: !timedelta "24:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     onestep: .true.
     cplmode: nems_frac
diff --git a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml
index ba69708cdf..b0d955bffa 100644
--- a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml
+++ b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C192.yaml
@@ -17,4 +17,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml
index d8071f420e..f199502ac4 100644
--- a/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml
+++ b/workflow/cases/public_release_v1/AtmRiv_2016021000_GFS@C384.yaml
@@ -17,4 +17,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml
index 7a9b56e8d8..b2a8cbef05 100644
--- a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml
+++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C192.yaml
@@ -19,4 +19,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml
index 6fbebe28f4..0726a92c3c 100644
--- a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml
+++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384-override-stuff-in-case-file.yaml
@@ -19,7 +19,7 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
 
   suite_overrides: !FirstTrue
     - when: !calc (doc.platform.name=="JET")
diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml
index 2787710cef..4f0303d8e2 100644
--- a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml
+++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C384.yaml
@@ -19,4 +19,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml
index 3146d4a0cd..f745f228d3 100644
--- a/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml
+++ b/workflow/cases/public_release_v1/Harvey_2017081712_GFS@C768.yaml
@@ -19,4 +19,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml
index e0221886dd..406839e5e1 100644
--- a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml
+++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C192.yaml
@@ -17,4 +17,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml
index 224b24c2a8..2afee405c5 100644
--- a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml
+++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C384.yaml
@@ -17,4 +17,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml
index 1a7e6da8e3..6f5a0bf0af 100644
--- a/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml
+++ b/workflow/cases/public_release_v1/March_Noreasters_2018022800_GFS@C768.yaml
@@ -17,4 +17,4 @@ case:
     IC_CDUMP: gfs
     run_gsi: No
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/tutorial_cycled.yaml b/workflow/cases/tutorial_cycled.yaml
index f426f45a83..18084941dc 100644
--- a/workflow/cases/tutorial_cycled.yaml
+++ b/workflow/cases/tutorial_cycled.yaml
@@ -17,4 +17,4 @@ case:
     KEEPDATA: Yes
     run_gsi: Yes
     chgres_and_convert_ics: No
-    gfs_cyc: 4 # run gfs every cycle
+    STEP_GFS: !timedelta "06:00:00"
diff --git a/workflow/cases/tutorial_free_forecast.yaml b/workflow/cases/tutorial_free_forecast.yaml
index e187acb506..e38d612945 100644
--- a/workflow/cases/tutorial_free_forecast.yaml
+++ b/workflow/cases/tutorial_free_forecast.yaml
@@ -9,11 +9,12 @@ case:
   settings: 
     SDATE: 2019-05-01t00:00:00
     EDATE: 2019-05-01t06:00:00
+    STEP_GFS: !timedelta "06:00:00"
+    STEP_DA: !timedelta "00:00:00"
 
     DUMP_SUFFIX: "p"
     run_gsi: No
     chgres_and_convert_ics: Yes
-    gfs_cyc: 4 # run gfs every cycle
 
   suite_overrides: !FirstTrue
     - when: !calc (doc.platform.name=="JET")
diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml
index 92c2457857..65bf998f6f 100644
--- a/workflow/config/base.yaml
+++ b/workflow/config/base.yaml
@@ -191,10 +191,7 @@ config_base:
     export FHMIN={doc.output_settings.FHMIN_GDAS}
     export FHMAX={doc.output_settings.FHMAX_GDAS}
     export FHOUT={doc.output_settings.FHOUT_GDAS}
-    
-    # GFS cycle info
-    export gfs_cyc={doc.settings.gfs_cyc} # 0: no GFS cycle, 1: 00Z only, 2: 00Z and 12Z only, 4: all 4 cycles.
-    
+        
     # GFS output and frequency
     export FHMIN_GFS={doc.output_settings.FHMIN_GFS}
     export FHMAX_GFS={doc.output_settings.FHMAX_GFS}
diff --git a/workflow/config/wave.yaml b/workflow/config/wave.yaml
index eeca0651da..732dabedb4 100644
--- a/workflow/config/wave.yaml
+++ b/workflow/config/wave.yaml
@@ -78,12 +78,8 @@ config_wave:
         WAVNCYC=4
         WAVHCYC=6
         FHMAX_WAV_CUR=48 # RTOFS forecasts only out to 8 days
-      elif [ $gfs_cyc -ne 0 ]
-      then
-        FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days
-        WAVHCYC=`expr 24 / $gfs_cyc`
       else
-        WAVHCYC=0
+        WAVHCYC={tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]}
         FHMAX_WAV_CUR=192 # RTOFS forecasts only out to 8 days
       fi
       export FHMAX_WAV_CUR WAVHCYC WAVNCYC
diff --git a/workflow/defaults/settings.yaml b/workflow/defaults/settings.yaml
index f6e52d3742..d954f925d8 100644
--- a/workflow/defaults/settings.yaml
+++ b/workflow/defaults/settings.yaml
@@ -35,7 +35,6 @@ default_settings: &default_settings
   ecflow_virtual_clock: no
   ecflow_hybrid_clock: no
 
-  gfs_cyc: 4
   nems_temp: 'atm'
   nems_temp_cold: 'atm'
   ics_from: opsgfs
@@ -78,6 +77,9 @@ default_settings: &default_settings
   IAUFHRS: 6
   IAU_DELTHRS: 6
 
+  STEP_DA: !timedelta "06:00:00"
+  STEP_GFS: !timedelta "06:00:00"
+
   KEEPDATA: NO
   DUMP_SUFFIX: ""
   run_gsi: No
diff --git a/workflow/layout/cycled_gfs.yaml b/workflow/layout/cycled_gfs.yaml
index 8c39f26769..216060a5e7 100644
--- a/workflow/layout/cycled_gfs.yaml
+++ b/workflow/layout/cycled_gfs.yaml
@@ -334,7 +334,7 @@ suite: !Cycle
   ######################################################################
 
   gfs: !Family
-    Disable: !calc doc.settings.gfs_cyc==0
+    Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
     Complete: !Depend ~ suite.has_cycle('-6:00:00')
     AlarmName: gfs
     RUN: 'gfs'
@@ -731,7 +731,7 @@ suite: !Cycle
 
     jgfs_archive: !Task
       <<: *service_task_template
-      Disable: !calc doc.settings.gfs_cyc == 0
+      Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
       Complete: !Depend ~ suite.has_cycle('-6:00:00')
       AlarmName: gfs
       Trigger: !Depend up.gfs
diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml
index 2d8f64da65..a0102240f4 100644
--- a/workflow/layout/free_forecast_gfs.yaml
+++ b/workflow/layout/free_forecast_gfs.yaml
@@ -12,6 +12,7 @@ suite: !Cycle
     Disable: !calc >-
       doc.settings.get("IC_CDUMP","") and not doc.settings.chgres_and_convert_ics or doc.settings.mom6ic_prepared=='.true.'
 #    RUN: 'gfs'
+    AlarmName: gfs
     jgfs_emc_getics: !Task
       <<: *service_task_template
       RUN: 'gfs'
@@ -50,6 +51,7 @@ suite: !Cycle
   gfs: !Family
 #    AlarmName: gdas                    ##!!"gdas" mean to run every 6 hrs from sdate to edate!!"
     RUN: 'gfs'
+    AlarmName: gfs
     ecflow_def: "edit RUN 'gfs'"
 
     prep: !Family
@@ -223,7 +225,7 @@ suite: !Cycle
 
     jgfs_archive: !Task
       <<: *service_task_template
-      Disable: !calc doc.settings.gfs_cyc == 0
+      Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
       AlarmName: gfs
       Trigger: !Depend up.gfs
       resources: !calc partition.resources.run_arch
@@ -241,6 +243,7 @@ suite: !Cycle
   final: !Task
     <<: *service_task_template
     Disable: !calc not metasched.type=="rocoto"
+    AlarmName: gfs
     resources: !calc partition.resources.run_final
     rocoto_command: /bin/true
     model: gfs # useless but required
diff --git a/workflow/layout/manual_temp.yaml b/workflow/layout/manual_temp.yaml
index 58696fbf71..139a4a54e0 100644
--- a/workflow/layout/manual_temp.yaml
+++ b/workflow/layout/manual_temp.yaml
@@ -243,7 +243,7 @@ suite: !Cycle
   ######################################################################
 
   gfs: !Family
-    Disable: !calc doc.settings.gfs_cyc==0
+    Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
     Complete: !Depend ~ suite.has_cycle('-6:00:00')
     AlarmName: gfs
     RUN: 'gfs'
@@ -503,7 +503,7 @@ suite: !Cycle
 
     jgfs_archive: !Task
       <<: *service_task_template
-      Disable: !calc doc.settings.gfs_cyc == 0
+      Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
       Complete: !Depend ~ suite.has_cycle('-6:00:00')
       AlarmName: gfs
       Trigger: !Depend up.gfs
diff --git a/workflow/layout/nco.yaml b/workflow/layout/nco.yaml
index ad9d82cb11..a2a95d52b8 100644
--- a/workflow/layout/nco.yaml
+++ b/workflow/layout/nco.yaml
@@ -280,7 +280,7 @@ suite: !Cycle
   ######################################################################
 
   gfs: !Family
-    Disable: !calc doc.settings.gfs_cyc==0
+    Disable: !calc doc.settings.STEP_GFS == doc.zero_dt
     Complete: !Depend ~ suite.has_cycle('-6:00:00')
     AlarmName: gfs
     RUN: 'gfs'
diff --git a/workflow/runtime/suite.yaml b/workflow/runtime/suite.yaml
index fd521f081d..17f51ba51f 100644
--- a/workflow/runtime/suite.yaml
+++ b/workflow/runtime/suite.yaml
@@ -2,6 +2,9 @@
 # suites in the workflow/ directory.  Most of these variables are
 # specific to the cycled workflow which is not in this release.
 
+day_dt: !timedelta "24:00:00"
+zero_dt: !timedelta "00:00:00"
+
 # gfs_clock_1 - run GFS only at 0 UTC daily
 gfs_clock_1: !Clock
   start: !calc tools.day_of(suite.Clock.start+suite.Clock.step)
@@ -13,12 +16,12 @@ gfs_clock_2: !Clock
   step: !timedelta "12:00:00"
 
 # gfs_clock_4 - run GFS every cycle
-gfs_clock_4: !Clock
-  start: !FirstTrue
-      - when: !calc doc.places.workflow_file == "layout/free_forecast_gfs.yaml"
-        do: !calc suite.Clock.start
-      - otherwise: !calc suite.Clock.start+suite.Clock.step
-  step: !timedelta "06:00:00"
+# gfs_clock_4: !Clock
+#   start: !FirstTrue
+#       - when: !calc doc.settings.STEP_DA == doc.zero_dt
+#         do: !calc suite.Clock.start
+#       - otherwise: !calc suite.Clock.start+suite.Clock.step
+#   step: !timedelta "06:00:00"
 
 # cycled_suite_alarms - this is used to define which cycles various
 # jobs run in.  In the workflow suite definition, jobs may have an
@@ -39,20 +42,26 @@ cycled_suite_alarms: &cycled_suite_alarms
 
   # gfs - cycles for which the gfs is run
   gfs: !FirstTrue
-    - when: !calc doc.settings.gfs_cyc == 0
+    # No free forecast
+    - when: !calc doc.settings.STEP_GFS == doc.zero_dt
       do: !Clock  # required but ignored because gfs is disabled
         start: !calc suite.Clock.start
         end: !calc suite.Clock.end
         step: !calc suite.Clock.step
-    - when: !calc doc.settings.gfs_cyc == 1
-      do: !calc suite.Clock.for_alarm(doc.gfs_clock_1)
-    - when: !calc doc.settings.gfs_cyc == 2
-      do: !calc suite.Clock.for_alarm(doc.gfs_clock_2)
-    - when: !calc doc.settings.gfs_cyc == 4
-      do: !calc suite.Clock.for_alarm(doc.gfs_clock_4)
-    - otherwise: !error Unknown gfs clock settings {doc.settings.gfs_cyc}
+    - otherwise: !Clock
+        start: !FirstTrue
+          - when: !calc doc.settings.STEP_DA == doc.zero_dt
+            # No DA
+            do: !calc suite.Clock.start
+            # Run one DA-only cycle before starting GFS
+          - otherwise: !calc suite.Clock.start+suite.Clock.step
+        end: !calc suite.Clock.end
+        step: !calc doc.settings.STEP_GFS
 
-  gfs_00_12: !calc suite.Clock.for_alarm(doc.gfs_clock_2)
+  gfs_00_12: !FirstTrue
+    - when: !calc doc.settings.STEP_GFS >= doc.day_dt
+      do: !calc suite.Clock.for_alarm(doc.gfs_clock_1)
+    - otherwise: !calc suite.Clock.for_alarm(doc.gfs_clock_2)
 
 
 # suite_defaults - this is inherited by the actual suite definitions
@@ -60,10 +69,14 @@ cycled_suite_alarms: &cycled_suite_alarms
 suite_defaults: &suite_defaults
   # Clock - the list of cycles to run
   Clock: !Clock
-    # NEED TO FIX FOR LESS FREQUENT GFS
     start: !calc doc.settings.SDATE
     end: !calc doc.settings.EDATE
-    step: !timedelta "6:00:00"
+    step: !FirstTrue
+      - when: !calc doc.settings.STEP_DA == doc.zero_dt
+        # No DA, use GFS step
+        do: !calc doc.settings.STEP_GFS
+        # Use DA step, which might be shorter than GFS step
+      - otherwise: !calc doc.settings.STEP_DA
 
   Overrides:
     allowed: [ 'queue', 'partition', 'resources', 'rocoto_command', 'rocoto_log_path',
diff --git a/workflow/schema/settings.yaml b/workflow/schema/settings.yaml
index 7606d59234..ec2cf23096 100644
--- a/workflow/schema/settings.yaml
+++ b/workflow/schema/settings.yaml
@@ -1,15 +1,4 @@
 settings_schema: !Template &settings_schema
-  gfs_cyc:
-    type: int
-    allowed: [ 0, 1, 2, 4 ]
-    description: |
-      When to run GFS forecast.  Data assimilation is run for every
-      cycle regardless of these values
-        * 0: no GFS cycle
-        * 1: 00Z only
-        * 2: 00Z and 12Z only
-        * 4: all 4 cycles (0, 6, 12, 18 Z)
-
   dev_safeguards:
     type: bool
     description: "Add backup triggers in workflow to handle scheduling delays, such as the forecast finishing any posts start.  Only turn this off for NCO operational deliveries."
@@ -99,6 +88,18 @@ settings_schema: !Template &settings_schema
   EDATE:
     type: datetime
     description: Last cycle to run GDAS.
+
+  STEP_DA:
+    type: timedelta
+    optional: true
+    description: |
+      Cadence to run DA.
+  
+  STEP_GFS:
+    type: timedelta
+    optional: true
+    description: |
+      Cadence to run GFS.
   
   ics_from:
     type: string

From 45def8c39f64afd809df9b7178d07d73f0161030 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 7 Jun 2021 20:47:51 +0000
Subject: [PATCH 2/6] Add restart output for aerosol cycling

Updates the restart_interval setting to make sure it always includes the
STEP_GFS time if GOCART is on. This is done using the advanced functionality
of restart_interval, which allows for an explicit list of restart times
instead of a single interval.

If STEP_GFS is a multiple of restart_interval, it is left unchanged.
If STEP_GFS is not a multiple of restart_interval, restart_interval is
converted to an explicit sequence of restart times, then STEP_GFS is added.
If restart_interval is already an explicit list, STEP_GFS is just added to it.
This may add a redundant restart time, but that is ignored by FV3.

In order to complete this update, the CROW type for restart_interval had to be
changed from int to string to accomodate lists of numbers. This has the added
benefit of allowing explicit definition of restart times in other applications
as well.

As part of this update, the values for restart_interval in the forecast config
were updated to use the appropriate settings rather than hardcoded values.
The default value for the gfs restart_interval was also changed from 6 to 0 to
maintain the previous behavior when the setting was ignored.

Refs: #314
---
 workflow/config/fcst.yaml       | 27 ++++++++++++++++++++++++---
 workflow/defaults/fv3_enkf.yaml |  2 +-
 workflow/defaults/fv3_gdas.yaml |  2 +-
 workflow/defaults/fv3_gfs.yaml  |  2 +-
 workflow/schema/fv3.yaml        |  2 +-
 5 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml
index d96dcac912..0ca769f484 100644
--- a/workflow/config/fcst.yaml
+++ b/workflow/config/fcst.yaml
@@ -39,6 +39,27 @@ config_fcst:
       do: !calc doc.partition_common.resources.run_coupled_fcst
     - otherwise: !calc doc.partition_common.resources.run_gfsfcst
 
+  restart_interval_gfs: !FirstTrue
+    - when: !calc doc.settings.cplgocart == ".false."
+      do: !calc doc.fv3_gfs_settings.restart_interval
+    - otherwise: !FirstTrue
+      - when: !calc doc.fv3_gfs_settings.restart_interval == "0"
+        # Add restart file at forecast cadence in addition to end of forecast
+        do: !calc tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0] + " " + str(doc.output_settings.FHMAX_GFS)
+      - when: !calc len(doc.fv3_gfs_settings.restart_interval.split(' ')) == 1
+        # Make sure forecast cadence is included in restart interval
+        do: !FirstTrue
+          # STEP_GFS is already a multiple of the restart interval, don't change
+          - when: !calc int(tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0])%int(doc.fv3_gfs_settings.restart_interval) == 0
+            do: !calc doc.fv3_gfs_settings.restart_interval
+          # STEP_GFS is not a multiple of the restart interval, convert to explicit list and add
+          - otherwise: !calc tools.join(tools.seq(0,doc.output_settings.FHMAX_GFS,int(doc.fv3_gfs_settings.restart_interval)) + [tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]], ' ')
+      # restart_interval is already an explicit list, make sure STEP_GFS is in the list
+      - otherwise: !FirstTrue
+        - when: !calc doc.fv3_gfs_settings.restart_interval.split(' ')[-1] == "-1"
+          do: !calc tools.join(doc.fv3_gfs_settings.restart_interval.split(' ')[:-1], ' ') + " " + tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]
+        - otherwise: !calc doc.fv3_gfs_settings.restart_interval + " " + tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]
+
   content: !expand |
     #!/bin/ksh -x
     
@@ -207,7 +228,7 @@ config_fcst:
         fi
     
         # Write restart files at next assimilation time
-        export restart_interval=6
+        export restart_interval={doc.fv3_enkf_settings.restart_interval}
         export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks}
         export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS}
 
@@ -219,10 +240,9 @@ config_fcst:
         else
             export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_orig"
         fi
+        export restart_interval={restart_interval_gfs}
         export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks}
         export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS}
-
-
     fi
 
     if [ $cplgocart = .true. ]; then # temporary settings for aerosol coupling
@@ -231,6 +251,7 @@ config_fcst:
         export CHM_CFGDIR="{doc.chem_settings.CFGDIR}"
         export CHM_INPDIR="{doc.chem_settings.INPDIR}"
         export dnats="$(( $dnats + {doc.chem_settings.ntdiag} ))"
+        export KMP_INIT_AT_FORK=FALSE # Temporary setting
     elif [[ $cpl == .true. ]] ; then # coupled model
         export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_cpl"
     fi
diff --git a/workflow/defaults/fv3_enkf.yaml b/workflow/defaults/fv3_enkf.yaml
index 1309ff4370..4128013773 100644
--- a/workflow/defaults/fv3_enkf.yaml
+++ b/workflow/defaults/fv3_enkf.yaml
@@ -120,7 +120,7 @@ fv3_enkf_defaults: &fv3_enkf_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "6"
   imp_physics: "11"
 #     if_present: !FirstTrue
 #      - when: !calc ( imp_physics==99 )
diff --git a/workflow/defaults/fv3_gdas.yaml b/workflow/defaults/fv3_gdas.yaml
index 9f4ea47976..5e02311697 100644
--- a/workflow/defaults/fv3_gdas.yaml
+++ b/workflow/defaults/fv3_gdas.yaml
@@ -93,7 +93,7 @@ fv3_gdas_defaults: &fv3_gdas_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "6"
   imp_physics: "11"
   phy_dependent_var: !Select
      select: !calc imp_physics
diff --git a/workflow/defaults/fv3_gfs.yaml b/workflow/defaults/fv3_gfs.yaml
index 8b7a7af7f4..e241a93c05 100644
--- a/workflow/defaults/fv3_gfs.yaml
+++ b/workflow/defaults/fv3_gfs.yaml
@@ -98,7 +98,7 @@ fv3_gfs_defaults: &fv3_gfs_defaults
   ICO2: 2
   warm_start: true
   read_increment: no
-  restart_interval: 6
+  restart_interval: "0"
   imp_physics: "11"
   phy_dependent_var: !Select
      select: !calc imp_physics
diff --git a/workflow/schema/fv3.yaml b/workflow/schema/fv3.yaml
index 61aaba6eb3..acdea9ef9e 100644
--- a/workflow/schema/fv3.yaml
+++ b/workflow/schema/fv3.yaml
@@ -53,7 +53,7 @@ fv3_settings_template: !Template &fv3_settings_template
   ICO2:               { type: int }
   warm_start:         { type: bool }
   read_increment:     { type: bool }
-  restart_interval:   { type: int }
+  restart_interval:   { type: string }
 #  FIELD_TABLE:        { type: string }
 
   LEVS:

From 84101b023f27eb0797868a52a9950bf16709bb89 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 7 Jun 2021 22:20:21 +0000
Subject: [PATCH 3/6] Add init_chem job to coupled workflow

A new init_chem job is added to the coupled workflow to cycle tracer
concentrations from the previous cycle's restart files. The new job
doesn't run for the first cycle (since there are no previous cycle
restart files to use). If aerosol initialization is desired for the
first cycle, they need to be added manually, though the new scripts
may be reutilized to do so.

Tracer concentrations are merely copied from the restart files to the
cold-start initial conditions. There is no adjustment to ensure mass
conservation or any other property. The list of tracers copied is
determined by the new parm/chem/gocart_tracer.list file, which is a
plain-text list of NetCDF variables to copy. If any are missing in the
restart files, the job will fail.

The NetCDF4 module for python3 is required for the init job to work.
This module is not part of the default installation of python on Hera
(though it is on WCOSS), so in addition to now loading python, the
base modulefile also adds the location of a personal copy of the
needed python module to the PYTHONPATH.

Most of the new scripts are recycled from the GEFS-aerosol development,
though this method was abandoned there in favor of true warm start for
the aerosol member.

Refs: #314
---
 jobs/rocoto/init_chem.sh                 |  56 +++++++++
 modulefiles/module_base.hera             |   9 +-
 parm/chem/gocart_tracer.list             |  20 ++++
 scripts/exgfs_chem_init_aerosol.py       | 137 +++++++++++++++++++++++
 ush/merge_fv3_chem_tile.py               | 116 +++++++++++++++++++
 workflow/config/init_chem.yaml           |  22 ++++
 workflow/defaults/default_resources.yaml |   7 ++
 workflow/layout/free_forecast_gfs.yaml   |  13 +++
 8 files changed, 378 insertions(+), 2 deletions(-)
 create mode 100755 jobs/rocoto/init_chem.sh
 create mode 100644 parm/chem/gocart_tracer.list
 create mode 100755 scripts/exgfs_chem_init_aerosol.py
 create mode 100755 ush/merge_fv3_chem_tile.py
 create mode 100644 workflow/config/init_chem.yaml

diff --git a/jobs/rocoto/init_chem.sh b/jobs/rocoto/init_chem.sh
new file mode 100755
index 0000000000..606cc09a9b
--- /dev/null
+++ b/jobs/rocoto/init_chem.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -x
+
+###############################################################
+## Abstract:
+## Create FV3 initial conditions from GFS intitial conditions
+## RUN_ENVIR : runtime environment (emc | nco)
+## HOMEgfs   : /full/path/to/workflow
+## EXPDIR : /full/path/to/config/files
+## CDATE  : current date (YYYYMMDDHH)
+## CDUMP  : cycle name (gdas / gfs)
+## PDY    : current date (YYYYMMDD)
+## cyc    : current cycle (HH)
+###############################################################
+
+###############################################################
+# Source FV3GFS workflow modules
+. $USHgfs/load_fv3gfs_modules.sh
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+###############################################################
+# Source relevant configs
+configs="base"
+for config in $configs; do
+    . $EXPDIR/config.${config}
+    status=$?
+    [[ $status -ne 0 ]] && exit $status
+done
+
+###############################################################
+# Source machine runtime environment
+. $BASE_ENV/${machine}.env init_aerosol
+status=$?
+[[ $status -ne 0 ]] && exit $status
+
+# export DATAROOT="$RUNDIR/$CDATE/$CDUMP"
+# [[ ! -d $DATAROOT ]] && mkdir -p $DATAROOT
+# export DATA=${DATA:-${DATAROOT}/${jobid:?}}
+# mkdir -p $DATA
+# cd $DATA
+
+$SCRgfs/exgfs_chem_init_aerosol.py
+
+status=$?
+if [[ $status -ne 0 ]]; then
+    echo "FATAL ERROR: exgfs_chem_init_aerosol.py failed with error code $status"
+    exit $status
+fi
+
+##############################################################
+# Exit cleanly
+
+set +x
+exit 0
diff --git a/modulefiles/module_base.hera b/modulefiles/module_base.hera
index 383e8f9b45..aa910eb6db 100644
--- a/modulefiles/module_base.hera
+++ b/modulefiles/module_base.hera
@@ -4,7 +4,8 @@
 ##
 
 # Modules availble through hpc-stack
-module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+# module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack
+module use /scratch1/NCEPDEV/nems/Raffaele.Montuoro/dev/nasa/libs/hpc-stack/1.1.0-nasa/modulefiles/stack
 
 module load hpc/1.1.0
 module load hpc-intel/18.0.5.274
@@ -18,6 +19,7 @@ module load crtm 2.3.0
 module load prod_util/1.2.2
 module load grib_util/1.2.2
 module load ip/3.3.3
+
 module load sp/2.3.3
 module load w3nco/2.4.1
 module load bacio/2.4.1
@@ -32,7 +34,7 @@ module load hdf5/1.10.6
 module load esmf/8_1_0_beta_snapshot_27
 module load w3emc/2.7.3
 module load wgrib2/2.0.8
-setenv WGRIB2 wgrib2
+setenv "WGRIB2" "wgrib2"
 
 # Modules not under hpc-stack
 module load hpss/hpss
@@ -40,3 +42,6 @@ module load nco/4.9.1
 module load gempak/7.4.2
 module load ncl/6.5.0
 module load cdo/1.9.5
+module load intelpython/3.6.8
+# netCDF4 is not in the python installation on Hera
+append-path "PYTHONPATH" "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/python_packages"
diff --git a/parm/chem/gocart_tracer.list b/parm/chem/gocart_tracer.list
new file mode 100644
index 0000000000..871a41eedb
--- /dev/null
+++ b/parm/chem/gocart_tracer.list
@@ -0,0 +1,20 @@
+so2
+sulf
+dms
+msa
+pp25
+pp10
+bc1
+bc2
+oc1
+oc2
+dust1
+dust2
+dust3
+dust4
+dust5
+seas1
+seas2
+seas3
+seas4
+seas5
\ No newline at end of file
diff --git a/scripts/exgfs_chem_init_aerosol.py b/scripts/exgfs_chem_init_aerosol.py
new file mode 100755
index 0000000000..75e615894f
--- /dev/null
+++ b/scripts/exgfs_chem_init_aerosol.py
@@ -0,0 +1,137 @@
+#! /usr/bin/env python3
+
+"""
+"""
+
+import os
+import subprocess
+import typing
+from datetime import datetime, timedelta
+from functools import partial
+
+# Constants
+atm_base_pattern = "{ics_dir}/%Y%m%d%H/atmos/{case}/INPUT"                    # Location of atmosphere ICs
+atm_file_pattern = "{path}/gfs_data.{tile}.nc"                                # Atm IC file names
+tracer_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/atmos/RERUN_RESTART"        # Time of previous run
+tracer_file_pattern = "{tracer_base}/%Y%m%d.%H0000.fv_tracer.res.{tile}.nc"   # Time when restart is valid (current run)
+tracer_list_file_pattern = "{parm_gfs}/chem/gocart_tracer.list"               # Text list of tracer names to copy
+merge_script_pattern = "{ush_gfs}/merge_fv3_chem_tile.py"
+n_tiles = 6
+max_lookback = 4                                                              # Maximum number of past cycles to look for for tracer data
+debug = True
+
+# End configurable settings
+
+# Make sure print statements are flushed immediately, otherwise
+#   print statments may be out-of-order with subprocess output
+print = partial(print, flush=True)
+
+tiles = list(map(lambda t: "tile{t}".format(t=t), range(1, n_tiles + 1)))
+
+
+def main() -> None:
+	# Read in environment variables and make sure they exist
+	cdate = get_env_var("CDATE")
+	incr = int(get_env_var('FHCYC'))
+	cdump = get_env_var("CDUMP")
+	rot_dir = get_env_var("ROTDIR")
+	ics_dir = get_env_var("ICSDIR")
+	case = get_env_var("CASE")
+	ush_gfs = get_env_var("USHgfs")
+	parm_gfs = get_env_var("PARMgfs")
+	# data = get_env_var('DATA')
+
+	# os.chdir(data)
+
+	merge_script = merge_script_pattern.format(ush_gfs=ush_gfs)
+	tracer_list_file = tracer_list_file_pattern.format(parm_gfs=parm_gfs)
+
+	time = datetime.strptime(cdate, "%Y%m%d%H")
+	atm_source_path = time.strftime(atm_base_pattern.format(**locals()))
+
+	if(debug):
+		for var in ['merge_script', 'tracer_list_file', 'atm_source_path']:
+			print(f'{var} = {f"{var}"}')
+
+	atm_files = get_atm_files(atm_source_path)
+	tracer_files = get_tracer_files(time, incr, max_lookback, rot_dir, cdump)
+
+	if (tracer_files is not None):
+		merge_tracers(merge_script, atm_files, tracer_files, tracer_list_file)
+
+	return
+
+
+# Retrieve environment variable and exit or print warning if not defined
+def get_env_var(varname: str, fail_on_missing: bool = True) -> str:
+	if(debug):
+		print(f'Trying to read envvar {varname}')
+
+	var = os.environ.get(varname)
+	if(var is None):
+		if(fail_on_missing is True):
+			print("FATAL: Environment variable {varname} not set".format(varname=varname))
+			exit(100)
+		else:
+			print("WARNING: Environment variable {varname} not set, continuing using None".format(varname=varname))
+	if(debug):
+		print(f'\tValue: {var}')
+	return(var)
+
+
+# Check if atm files exist
+def get_atm_files(path: str) -> typing.List[str]:
+	print(f'Checking for atm files in {path}')
+
+	files = list(map(lambda tile: atm_file_pattern.format(tile=tile, path=path), tiles))
+	for file_name in files:
+		if(debug):
+			print(f"\tChecking for {file_name}")
+		if(not os.path.isfile(file_name)):
+			print("FATAL: Atmosphere file {file_name} not found".format(file_name=file_name))
+			exit(101)
+		elif(debug):
+			print(f"\t\tFound {file_name}")
+	return files
+
+
+# Find last cycle with tracer data available via restart files
+def get_tracer_files(time: datetime, incr: int, max_lookback: int, rot_dir: str, cdump: str) -> typing.List[str]:
+	print(f"Looking for tracer files in {rot_dir}")
+	for lookback in map(lambda i: incr * (i + 1), range(max_lookback)):
+		last_time = time - timedelta(hours=lookback)
+		if(debug):
+			print(f"\tChecking {last_time}")
+		tracer_base = last_time.strftime(tracer_base_pattern.format(**locals()))
+		files = list(map(lambda tile: time.strftime(tracer_file_pattern.format(tracer_base=tracer_base, tile=tile)), tiles))
+		if(debug):
+			print("\t\tLooking for files {files} in directory {tracer_base}")
+		found = [file for file in files if os.path.isfile(file)]
+		if(found):
+			if(debug):
+				print(f"\t\tAll files found!")
+			return files
+		else:
+			print(last_time.strftime("Tracer files not found for %Y%m%d_%H"))
+
+	if(not found):
+		print("WARNING: Unable to find tracer files, will use zero fields")
+		return None
+
+
+# Merge tracer data into atmospheric data
+def merge_tracers(merge_script: str, atm_files: typing.List[str], tracer_files: typing.List[str], tracer_list_file: str) -> None:
+	print(f"Merging tracers")
+	if(len(atm_files) != len(tracer_files)):
+		print("FATAL: atmosphere file list and tracer file list are not the same length")
+		exit(102)
+
+	for atm_file, tracer_file in zip(atm_files, tracer_files):
+		if(debug):
+			print(f"\tMerging tracers from {tracer_file} into {atm_file}")
+		subprocess.run([merge_script, atm_file, tracer_file, tracer_list_file], check=True)
+
+
+if __name__ == "__main__":
+	main()
+	exit(0)
diff --git a/ush/merge_fv3_chem_tile.py b/ush/merge_fv3_chem_tile.py
new file mode 100755
index 0000000000..e973f48c38
--- /dev/null
+++ b/ush/merge_fv3_chem_tile.py
@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Appends tracer data from one NetCDF file to another and updates the tracer
+count.
+
+usage: merge_fv3_chem_tile.py [-h] atm_file chem_file variable_file [out_file]
+
+Appends tracer data from one NetCDF file to another and updates the tracer
+count.
+
+positional arguments:
+  atm_file       File containing the atmospheric data
+  chem_file      File containing the chemistry tracer data to be added
+  variable_file  File containing list of tracer variable_names in the
+                  chem_file to add to the atm_file, one tracer per line
+  out_file       Name of file to create. If none is specified, the atm_file
+                  will be edited in place. New file will be a copy of atm_file
+                  with the specificed tracers listed in variable_file appended
+                  from chem_file and ntracers updated.
+
+optional arguments:
+  -h, --help     show this help message and exit
+
+"""
+
+import os, sys, subprocess
+from typing import List
+from functools import partial
+from shutil import copyfile
+import argparse
+
+try:
+	import netCDF4
+except ImportError:
+	print("FATAL ERROR: Failed to import netCDF4 in " + __file__ + "!")
+	print("Make sure you are using a version of python 3 with netCDF4 installed")
+	sys.exit(100)
+
+# Make sure print statements are flushed immediately, otherwise
+#   print statments may be out-of-order with subprocess output
+print = partial(print, flush=True)
+
+def merge_tile(base_file_name: str, append_file_name: str, tracers_to_append: List[str]) -> None:
+	if not os.path.isfile(base_file_name):
+		print("FATAL ERROR: Atmosphere file " + base_file_name + " does not exist!")
+		sys.exit(102)
+
+	if not os.path.isfile(append_file_name):
+		print("FATAL ERROR: Chemistry file " + append_file_name + " does not exist!")
+		sys.exit(103)
+
+	append_file = netCDF4.Dataset(append_file_name, "r")
+	base_file = netCDF4.Dataset(base_file_name, "r+")
+
+	# print(base_file)
+	# print(base_file.dimensions["ntracer"])
+
+	old_ntracer = base_file.dimensions["ntracer"].size
+	new_ntracer = old_ntracer + len(tracers_to_append)
+
+	# Copy over chemistry dimensions
+	for dim_name in append_file.dimensions:
+		base_file.createDimension(dim_name, append_file.dimensions[dim_name].size)
+
+	for variable_name in tracers_to_append:
+		print("Adding variable " + variable_name + " to file")
+		variable = append_file[variable_name]
+		base_file.createVariable(variable_name, variable.datatype, variable.dimensions)
+		base_file[variable_name][:] = variable[:]
+		base_file[variable_name].setncatts(variable.__dict__)
+		# print("Done adding " + variable_name)
+
+	print("Updating ntracer")
+
+	# Use ncks to rewrite file without ntracer so we can define it anew
+	subprocess.run(["ncks", "-x", "-v", "ntracer", "-O", base_file_name, base_file_name], check=True)
+
+	base_file = netCDF4.Dataset(base_file_name, "r+")
+	base_file.createDimension("ntracer", new_ntracer)
+	# print(base_file.dimensions["ntracer"])
+
+
+def main() -> None:
+	parser = argparse.ArgumentParser(
+		description="Appends tracer data from one NetCDF file to another and updates the tracer count.")
+	parser.add_argument('atm_file', type=str, help="File containing the atmospheric data")
+	parser.add_argument('chem_file', type=str, help="File containing the chemistry tracer data to be added")
+	parser.add_argument('variable_file', type=str, help="File containing list of tracer variable_names in the chem_file to add to the atm_file, one tracer per line")
+	parser.add_argument('out_file', type=str, nargs="?", help="Name of file to create. If none is specified, the atm_file will be edited in place. New file will be a copy of atm_file with the specificed tracers listed in variable_file appended from chem_file and ntracers updated.")
+
+	args = parser.parse_args()
+
+	atm_file_name = args.atm_file
+	chem_file_name = args.chem_file
+	variable_file = args.variable_file
+	out_file_name = args.out_file
+
+	if out_file_name is None:
+		print("INFO: No out_file specified, will edit atm_file in-place")
+		out_file_name = atm_file_name
+	else:
+		if os.path.isfile(out_file_name):
+			print("WARNING: Specified out file " + out_file_name + " exists and will be overwritten")
+		copyfile(atm_file_name, out_file_name)
+
+	variable_file = open(variable_file)
+	variable_names = variable_file.read().splitlines()
+	variable_file.close()
+
+	merge_tile(out_file_name, chem_file_name, variable_names)
+
+	# print(variable_names)
+
+
+if __name__ == "__main__":
+	main()
diff --git a/workflow/config/init_chem.yaml b/workflow/config/init_chem.yaml
new file mode 100644
index 0000000000..fff4a63d70
--- /dev/null
+++ b/workflow/config/init_chem.yaml
@@ -0,0 +1,22 @@
+# This file is used to generate config.init_chem
+
+config_init_chem:
+  filename: config.init_chem
+  content: !expand |
+    #!/bin/ksh -x
+    
+    # This file is automatically generated from the YAML-based system
+    # in ecf/ecfutils/.  Any changes will be overwritten if
+    # setup_case.sh is rerun.
+    
+    ########## config.init_chem ##########
+    # Append chem tracers from previous cycle to atmosphere initial conditions
+    
+    echo "BEGIN: config.init_chem"
+    
+    # Task and thread configuration
+    export npe_init_chem={doc.partition_common.resources.run_init_chem[0].mpi_ranks}
+    export npe_node_init_chem={doc.partition_common.resources.run_init_chem[0].mpi_ranks}
+    
+    echo "END: config.init_chem"
+    
diff --git a/workflow/defaults/default_resources.yaml b/workflow/defaults/default_resources.yaml
index 482991e292..fa4ac44dc8 100644
--- a/workflow/defaults/default_resources.yaml
+++ b/workflow/defaults/default_resources.yaml
@@ -117,6 +117,7 @@ util_resource_table:
       #                    ranks ppn  wallclock          threads  MB_batch MB_per_rank
   prepbufr:              [    4,    4, !timedelta "00:15:00",   1, 3072M,      null   ]
   fv3ic:                 [   24,   24, !timedelta "00:30:00",   1, 3072M,      null   ]
+  init_chem:             [    1,    1, !timedelta "00:05:00",   1,  600M,      null   ]
   wavepostpnt:           [   80,   40, !timedelta "01:00:00",   1, 3072M,      null   ]
   wavepostgrd:           [   10,   10, !timedelta "00:30:00",   1, 3072M,      null   ]
   dump_waiter:           [    1,    1, !timedelta "01:00:00",   1,  300M,      null   ]
@@ -194,6 +195,12 @@ default_resources: &default_resources
       max_ppn: !icalc doc.util_resource_table.fv3ic[1]
       walltime: !icalc doc.util_resource_table.fv3ic[2]
 
+  run_init_chem: !JobRequest
+    - batch_memory: !icalc doc.util_resource_table.init_chem[4]
+      mpi_ranks: !icalc doc.util_resource_table.init_chem[0]
+      max_ppn: !icalc doc.util_resource_table.init_chem[1]
+      walltime: !icalc doc.util_resource_table.init_chem[2]
+
   run_wavepostpnt: !JobRequest
     - batch_memory: !icalc doc.util_resource_table.wavepostpnt[4]
       mpi_ranks: !icalc doc.util_resource_table.wavepostpnt[0]
diff --git a/workflow/layout/free_forecast_gfs.yaml b/workflow/layout/free_forecast_gfs.yaml
index a0102240f4..fff63b344d 100644
--- a/workflow/layout/free_forecast_gfs.yaml
+++ b/workflow/layout/free_forecast_gfs.yaml
@@ -44,6 +44,17 @@ suite: !Cycle
       ecf_module_commands: "# fv3ic.sh will load modules instead"
       model: gfs
 
+    jgfs_init_chem: !Task
+      <<: *exclusive_task_template
+      RUN: 'gfs'
+      Disable: !calc doc.settings.cplgocart != '.true.'
+      Trigger: !Depend jgfs_emc_mom6ic & up.gfs.forecast.at('-' + tools.dt_to_HMS(doc.settings.STEP_GFS))
+      resources: !calc partition.resources.run_init_chem
+      J_JOB: rocoto/init_chem.sh
+      ecf_module_commands: "# init_chem.sh will load modules instead"
+      model: gfs
+      Complete: !Depend ~ suite.has_cycle('-' + tools.dt_to_HMS(doc.settings.STEP_GFS))
+
   #####################################################################
   ## GFS FAMILY #######################################################
   #####################################################################
@@ -88,6 +99,8 @@ suite: !Cycle
              do: !Depend medcold
            - when: !calc doc.settings.cplwav=='.true.'
              do: !Depend up.prep
+           - when: !calc doc.settings.cplgocart=='.true.'
+             do: !Depend up.up.ics.jgfs_init_chem & up.up.ics.jgfs_emc_mom6ic
            - otherwise: !Depend up.up.ics.jgfs_emc_mom6ic
         resources_remap: !JobRequest [ { <<: *remap_resource_template } ]
         resources: !FirstTrue

From 062888e0d045f97be157b559a43120c4ca6c983f Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 14 Jun 2021 18:31:26 +0000
Subject: [PATCH 4/6] Fix print statement in chem init

The print statement for the tracer (restart) file search was missing
an f, so variables were not being substituted.

Refs: #314
---
 scripts/exgfs_chem_init_aerosol.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/exgfs_chem_init_aerosol.py b/scripts/exgfs_chem_init_aerosol.py
index 75e615894f..11c1c50782 100755
--- a/scripts/exgfs_chem_init_aerosol.py
+++ b/scripts/exgfs_chem_init_aerosol.py
@@ -97,7 +97,7 @@ def get_atm_files(path: str) -> typing.List[str]:
 
 # Find last cycle with tracer data available via restart files
 def get_tracer_files(time: datetime, incr: int, max_lookback: int, rot_dir: str, cdump: str) -> typing.List[str]:
-	print(f"Looking for tracer files in {rot_dir}")
+	print(f"Looking for restart tracer files in {rot_dir}")
 	for lookback in map(lambda i: incr * (i + 1), range(max_lookback)):
 		last_time = time - timedelta(hours=lookback)
 		if(debug):
@@ -105,7 +105,7 @@ def get_tracer_files(time: datetime, incr: int, max_lookback: int, rot_dir: str,
 		tracer_base = last_time.strftime(tracer_base_pattern.format(**locals()))
 		files = list(map(lambda tile: time.strftime(tracer_file_pattern.format(tracer_base=tracer_base, tile=tile)), tiles))
 		if(debug):
-			print("\t\tLooking for files {files} in directory {tracer_base}")
+			print(f"\t\tLooking for files {files} in directory {tracer_base}")
 		found = [file for file in files if os.path.isfile(file)]
 		if(found):
 			if(debug):

From 19a8f96c041db57589c7535e2c1fbb4491e852ac Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 14 Jun 2021 18:41:23 +0000
Subject: [PATCH 5/6] Fix minor issue with restart interval

The restart_interval export needed to be enclosed in quotation marks
so bash will properly assign the whole list to the variable.

Refs: #314
---
 workflow/config/fcst.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/workflow/config/fcst.yaml b/workflow/config/fcst.yaml
index 0ca769f484..a89a9e2e84 100644
--- a/workflow/config/fcst.yaml
+++ b/workflow/config/fcst.yaml
@@ -240,7 +240,7 @@ config_fcst:
         else
             export DIAG_TABLE="$HOMEgfs/parm/parm_fv3diag/diag_table_orig"
         fi
-        export restart_interval={restart_interval_gfs}
+        export restart_interval="{restart_interval_gfs}"
         export npe_remap={doc.partition_common.resources.fallback_run_gfsremap[0].mpi_ranks}
         export nth_remap={doc.partition_common.resources.fallback_run_gfsremap[0].OMP_NUM_THREADS}
     fi

From c73040c177f15b21f1d09b0c861fdd6988931f65 Mon Sep 17 00:00:00 2001
From: "Walter.Kolczynski" <Walter.Kolczynski@noaa.gov>
Date: Mon, 14 Jun 2021 20:41:23 +0000
Subject: [PATCH 6/6] Correct cycling interval issues for aerosols

FHCYC was being used to determine the forecast interval for the aerosol
cycling, but this appears to be an appropriate setting to use. Added a
new exported environment variable STEP_GFS, which is the number of hours
between GFS forecasts. While it is derived from the workflow STEP_GFS
setting, it is only the integer number of hours, not a timedelta like
the workflow setting. May want to revist this later to avoid confusion.

Additionally, the restart file written at the end of the forecast does
not have the timestamp as part of the filename like the intermediary
restart files do. The chem init script is adjusted to not use the time-
stamp when looking for the final restart file. And, since it now has to
read in FHMAX anyway, also added a short-circuit so the search doesn't
bother trying to search previous cycles that can't include the restart
time needed.

Refs: #314
---
 scripts/exgfs_chem_init_aerosol.py | 23 +++++++++++++++++------
 workflow/config/base.yaml          |  1 +
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/scripts/exgfs_chem_init_aerosol.py b/scripts/exgfs_chem_init_aerosol.py
index 11c1c50782..fbc9cf094d 100755
--- a/scripts/exgfs_chem_init_aerosol.py
+++ b/scripts/exgfs_chem_init_aerosol.py
@@ -12,8 +12,8 @@
 # Constants
 atm_base_pattern = "{ics_dir}/%Y%m%d%H/atmos/{case}/INPUT"                    # Location of atmosphere ICs
 atm_file_pattern = "{path}/gfs_data.{tile}.nc"                                # Atm IC file names
-tracer_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/atmos/RERUN_RESTART"        # Time of previous run
-tracer_file_pattern = "{tracer_base}/%Y%m%d.%H0000.fv_tracer.res.{tile}.nc"   # Time when restart is valid (current run)
+tracer_base_pattern = "{rot_dir}/{cdump}.%Y%m%d/%H/atmos/RERUN_RESTART"       # Location of restart files (time of previous run)
+tracer_file_pattern = "{tracer_base}/{timestamp}fv_tracer.res.{tile}.nc"      # Name of restart files (time when restart is valid)
 tracer_list_file_pattern = "{parm_gfs}/chem/gocart_tracer.list"               # Text list of tracer names to copy
 merge_script_pattern = "{ush_gfs}/merge_fv3_chem_tile.py"
 n_tiles = 6
@@ -32,7 +32,8 @@
 def main() -> None:
 	# Read in environment variables and make sure they exist
 	cdate = get_env_var("CDATE")
-	incr = int(get_env_var('FHCYC'))
+	incr = int(get_env_var('STEP_GFS'))
+	fcst_length = int(get_env_var('FHMAX_GFS'))
 	cdump = get_env_var("CDUMP")
 	rot_dir = get_env_var("ROTDIR")
 	ics_dir = get_env_var("ICSDIR")
@@ -54,7 +55,7 @@ def main() -> None:
 			print(f'{var} = {f"{var}"}')
 
 	atm_files = get_atm_files(atm_source_path)
-	tracer_files = get_tracer_files(time, incr, max_lookback, rot_dir, cdump)
+	tracer_files = get_tracer_files(time, incr, max_lookback, fcst_length, rot_dir, cdump)
 
 	if (tracer_files is not None):
 		merge_tracers(merge_script, atm_files, tracer_files, tracer_list_file)
@@ -96,14 +97,24 @@ def get_atm_files(path: str) -> typing.List[str]:
 
 
 # Find last cycle with tracer data available via restart files
-def get_tracer_files(time: datetime, incr: int, max_lookback: int, rot_dir: str, cdump: str) -> typing.List[str]:
+def get_tracer_files(time: datetime, incr: int, max_lookback: int, fcst_length: int, rot_dir: str, cdump: str) -> typing.List[str]:
 	print(f"Looking for restart tracer files in {rot_dir}")
 	for lookback in map(lambda i: incr * (i + 1), range(max_lookback)):
+		if(lookback > fcst_length):
+			# Trying to look back farther than the length of a forecast
+			break
+		elif(lookback == fcst_length):
+			# Restart files at the end of the cycle don't have a timestamp
+			timestamp = ""
+		else:
+			timestamp = time.strftime("%Y%m%d.%H0000.")
+
 		last_time = time - timedelta(hours=lookback)
+
 		if(debug):
 			print(f"\tChecking {last_time}")
 		tracer_base = last_time.strftime(tracer_base_pattern.format(**locals()))
-		files = list(map(lambda tile: time.strftime(tracer_file_pattern.format(tracer_base=tracer_base, tile=tile)), tiles))
+		files = list(map(lambda tile: tracer_file_pattern.format(timestamp=timestamp, tracer_base=tracer_base, tile=tile), tiles))
 		if(debug):
 			print(f"\t\tLooking for files {files} in directory {tracer_base}")
 		found = [file for file in files if os.path.isfile(file)]
diff --git a/workflow/config/base.yaml b/workflow/config/base.yaml
index 65bf998f6f..f118f180d7 100644
--- a/workflow/config/base.yaml
+++ b/workflow/config/base.yaml
@@ -199,6 +199,7 @@ config_base:
     export FHMAX_HF_GFS={doc.output_settings.FHMAX_HF_GFS}
     export FHOUT_HF_GFS={doc.output_settings.FHOUT_HF_GFS}
     export FHRLST="{doc.output_settings.gfs_forecast_hours}"
+    export STEP_GFS="{tools.dt_to_HMS(doc.settings.STEP_GFS).split(':')[0]}"
 
     # I/O QUILTING, true--use Write Component; false--use GFDL FMS
     # if quilting=true, choose OUTPUT_GRID as cubed_sphere_grid in netcdf or gaussian_grid