From 512bf7cb8d1873185334d60fd05ee26a918745a9 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Tue, 30 Jan 2024 12:02:39 +0000 Subject: [PATCH] example: extending workflow --- .../etc/examples/extending-workflow/.validate | 128 ++++++++++++++++++ .../extending-workflow/complex/flow.cylc | 44 ++++++ .../etc/examples/extending-workflow/index.rst | 119 ++++++++++++++++ .../extending-workflow/simple/flow.cylc | 17 +++ 4 files changed, 308 insertions(+) create mode 100755 cylc/flow/etc/examples/extending-workflow/.validate create mode 100644 cylc/flow/etc/examples/extending-workflow/complex/flow.cylc create mode 100644 cylc/flow/etc/examples/extending-workflow/index.rst create mode 100644 cylc/flow/etc/examples/extending-workflow/simple/flow.cylc diff --git a/cylc/flow/etc/examples/extending-workflow/.validate b/cylc/flow/etc/examples/extending-workflow/.validate new file mode 100755 index 00000000000..3fea2ff7fdc --- /dev/null +++ b/cylc/flow/etc/examples/extending-workflow/.validate @@ -0,0 +1,128 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +test_simple () { + local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + + # lint + cylc lint ./simple + + # copy into a temp directory + local SRC_DIR="$(mktemp -d)" + cp simple/flow.cylc "$SRC_DIR" + + # speed things up with simulation mode + cat >>"${SRC_DIR}/flow.cylc" <<__HERE__ + [runtime] + [[root]] + [[[simulation]]] + default run length = PT0S +__HERE__ + + # start the workflow + cylc vip \ + --check-circular \ + --no-run-name \ + --no-detach \ + --workflow-name "$ID" \ + --mode=simulation \ + "$SRC_DIR" + + # it should have reached the 2002 cycle + grep '2002/a' "${HOME}/cylc-run/${ID}/log/scheduler/log" + ! grep '2003/a' "${HOME}/cylc-run/${ID}/log/scheduler/log" + + # edit the "stop after cycle point" + sed -i \ + 's/stop after cycle point.*/stop after cycle point = 2004/' \ + "${SRC_DIR}/flow.cylc" + + # continue the run + cylc vr \ + --no-detach \ + --mode=simulation \ + --yes \ + "$ID" + + # it should have reached the 2004 cycle + grep '2004/a' "${HOME}/cylc-run/${ID}/log/scheduler/log" + ! grep '2005/a' "${HOME}/cylc-run/${ID}/log/scheduler/log" + + # clean up + cylc clean "$ID" + + rm -r "${SRC_DIR}" +} + + +test_complex () { + local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)" + + # lint + cylc lint ./complex + + # copy into a temp directory + local SRC_DIR="$(mktemp -d)" + cp complex/flow.cylc "$SRC_DIR" + + # speed things up with simulation mode + cat >>"${SRC_DIR}/flow.cylc" <<__HERE__ + [runtime] + [[root]] + [[[simulation]]] + default run length = PT0S +__HERE__ + + # start the workflow + cylc vip \ + --check-circular \ + --no-run-name \ + --no-detach \ + --workflow-name "$ID" \ + --mode=simulation \ + -s stop_cycle=2003 \ + "$SRC_DIR" + + # it should have reached the 2003 cycle + grep '2003/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log" + grep '2003/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log" + ! grep '2002/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log" + + # continue the run + cylc vr \ + --no-detach \ + --mode=simulation \ + --yes \ + -s stop_cycle=2004 \ + "$ID" + + # it should have reached the 2004 cycle + grep '2004/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log" + grep '2004/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log" + ! grep '2005/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log" + + # clean up + cylc clean "$ID" + + rm -r "${SRC_DIR}" +} + + +# test_simple +test_complex diff --git a/cylc/flow/etc/examples/extending-workflow/complex/flow.cylc b/cylc/flow/etc/examples/extending-workflow/complex/flow.cylc new file mode 100644 index 00000000000..96664788d5a --- /dev/null +++ b/cylc/flow/etc/examples/extending-workflow/complex/flow.cylc @@ -0,0 +1,44 @@ +#!Jinja2 + +{# + override this default on the command line + e.g. "cylc play -s stop_cycle=2020" +#} +{% set stop_cycle = stop_cycle | default('2010') %} + +[scheduler] + # use the year for the cycle point + # (strip off the month, day, hour and minute) + cycle point format = CCYY + allow implicit tasks = True + +[scheduling] + initial cycle point = 2000 + stop after cycle point = {{ stop_cycle }} + [[graph]] + # first cycle only + R1 = """ + build => run_model + install => run_model + """ + + # first three cycles only + R3/2000/P1Y = """ + initial_conditions => run_model + """ + + # every cycle + P1Y = """ + run_model[-P1Y] => run_model + """ + + # last three cycles + R3/P1Y/{{ stop_cycle }} = """ + process[-P1Y] => process + run_model => process + """ + + # last cycle + R1/{{ stop_cycle }} = """ + process => analyse => plot + """ diff --git a/cylc/flow/etc/examples/extending-workflow/index.rst b/cylc/flow/etc/examples/extending-workflow/index.rst new file mode 100644 index 00000000000..154b07a6a58 --- /dev/null +++ b/cylc/flow/etc/examples/extending-workflow/index.rst @@ -0,0 +1,119 @@ +Extending Workflow +================== + +.. cylc:scope:: [scheduling] + +Sometimes we may run a workflow to completion, but subsequently wish to +run it for a few more cycles. + +With Cylc 7 this was often done by changing the `final cycle point` and +restarting the workflow. This approach worked, but was a little awkward. +It's possible with Cylc 8, but we would recommend moving away from this +pattern instead. + +The recommended approach to this problem (Cylc 6+) is to use the +`stop after cycle point` rather than the `final cycle point`. + +The `stop after cycle point` tells Cylc to **stop** after the workflow passes +the specified point, whereas the `final cycle point` tells Cylc that the +workflow **finishes** at the specified point. + +When a workflow **finishes**, it is a little awkward to restart as you have to +tell Cylc which tasks to continue on from. The `stop after cycle point` +solution avoids this issue. + + +Simple Example +-------------- + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/extending-workflow/simple + +This workflow will stop at the end of the ``2002`` cycle: + +.. literalinclude:: simple/flow.cylc + :language: cylc + +After it has run and shut down, change the `stop after cycle point` to +the desired value and restart it. E.g: + +.. code-block:: bash + + # install and run the workflow: + cylc vip + + # then later edit "stop after cycle point" to "2004" + + # then reinstall and restart the workflow: + cylc vr + +The workflow will continue from where it left off and run until the end of the +``2004`` cycle. Because the workflow never hit the `final cycle point` it +never "finished" so no special steps are required to restart the workflow. + +You can also set the `stop after cycle point` when you start the workflow: + +.. code-block:: bash + + cylc play --stop-cycle-point=2020 myworkflow + +Or change it at any point whilst the workflow is running: + +.. code-block:: bash + + cylc stop myworkflow//2030 # change the stop after cycle point to 2030 + +.. note:: + + If you set the `stop after cycle point` on the command line, this value will + take precedence over the one in the workflow configuration. Use + ``cylc play --stop-cycle-point=reload`` to restart the workflow using the + `stop after cycle point` configured in the workflow configuration. + + +Complex Example +--------------- + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/extending-workflow/complex + +Sometimes we may want to run some tasks at the `final cycle point` or in the +cycles leading up to the `final cycle point`. For example you might have some +analysis or housekeeping tasks to run at the end of the workflow. + +We can schedule such task like so: + +.. code-block:: cylc + + # run once in the "2000" cycle + R1/2000 # 2000 + + # run every year three times ending in 2000 + R3/P1Y/2000 # 1998, 1999, 2000 + +For more information see :ref:`user_guide.cycling_format_4`. + +This example uses Jinja2 to template the cycling expressions above to schedule +tasks to run at, and in the run up to, the `stop after cycle point`: + +.. literalinclude:: complex/flow.cylc + :language: cylc + +.. code-block:: bash + + # install and run the workflow: + cylc vip + + # then reinstall and restart the workflow extending the stop cycle: + cylc vr -s stop_cycle=2020 + + +.. cylc:scope:: diff --git a/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc new file mode 100644 index 00000000000..427c76101dd --- /dev/null +++ b/cylc/flow/etc/examples/extending-workflow/simple/flow.cylc @@ -0,0 +1,17 @@ +[scheduler] + # use the year for the cycle point + # (strip off the month, day, hour and minute) + cycle point format = CCYY + +[scheduling] + initial cycle point = 2000 + stop after cycle point = 2002 # stop after two years of simulated time + [[graph]] + P1Y = """ + z[-P1Y] => a + a => z + """ + +[runtime] + [[a]] + [[z]]