Skip to content

Commit 512bf7c

Browse files
oliver-sandersoliver-sanders
committed
example: extending workflow
1 parent 08d9c53 commit 512bf7c

File tree

4 files changed

+308
-0
lines changed

4 files changed

+308
-0
lines changed

‎cylc/flow/etc/examples/extending-workflow/.validate

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
#!/bin/bash
2+
# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
3+
# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
4+
#
5+
# This program is free software: you can redistribute it and/or modify
6+
# it under the terms of the GNU General Public License as published by
7+
# the Free Software Foundation, either version 3 of the License, or
8+
# (at your option) any later version.
9+
#
10+
# This program is distributed in the hope that it will be useful,
11+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
12+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13+
# GNU General Public License for more details.
14+
#
15+
# You should have received a copy of the GNU General Public License
16+
# along with this program. If not, see <http://www.gnu.org/licenses/>.
17+
18+
set -eux
19+
20+
test_simple () {
21+
local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
22+
23+
# lint
24+
cylc lint ./simple
25+
26+
# copy into a temp directory
27+
local SRC_DIR="$(mktemp -d)"
28+
cp simple/flow.cylc "$SRC_DIR"
29+
30+
# speed things up with simulation mode
31+
cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
32+
[runtime]
33+
[[root]]
34+
[[[simulation]]]
35+
default run length = PT0S
36+
__HERE__
37+
38+
# start the workflow
39+
cylc vip \
40+
--check-circular \
41+
--no-run-name \
42+
--no-detach \
43+
--workflow-name "$ID" \
44+
--mode=simulation \
45+
"$SRC_DIR"
46+
47+
# it should have reached the 2002 cycle
48+
grep '2002/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
49+
! grep '2003/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
50+
51+
# edit the "stop after cycle point"
52+
sed -i \
53+
's/stop after cycle point.*/stop after cycle point = 2004/' \
54+
"${SRC_DIR}/flow.cylc"
55+
56+
# continue the run
57+
cylc vr \
58+
--no-detach \
59+
--mode=simulation \
60+
--yes \
61+
"$ID"
62+
63+
# it should have reached the 2004 cycle
64+
grep '2004/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
65+
! grep '2005/a' "${HOME}/cylc-run/${ID}/log/scheduler/log"
66+
67+
# clean up
68+
cylc clean "$ID"
69+
70+
rm -r "${SRC_DIR}"
71+
}
72+
73+
74+
test_complex () {
75+
local ID="$(< /dev/urandom tr -dc A-Za-z | head -c6)"
76+
77+
# lint
78+
cylc lint ./complex
79+
80+
# copy into a temp directory
81+
local SRC_DIR="$(mktemp -d)"
82+
cp complex/flow.cylc "$SRC_DIR"
83+
84+
# speed things up with simulation mode
85+
cat >>"${SRC_DIR}/flow.cylc" <<__HERE__
86+
[runtime]
87+
[[root]]
88+
[[[simulation]]]
89+
default run length = PT0S
90+
__HERE__
91+
92+
# start the workflow
93+
cylc vip \
94+
--check-circular \
95+
--no-run-name \
96+
--no-detach \
97+
--workflow-name "$ID" \
98+
--mode=simulation \
99+
-s stop_cycle=2003 \
100+
"$SRC_DIR"
101+
102+
# it should have reached the 2003 cycle
103+
grep '2003/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log"
104+
grep '2003/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
105+
! grep '2002/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
106+
107+
# continue the run
108+
cylc vr \
109+
--no-detach \
110+
--mode=simulation \
111+
--yes \
112+
-s stop_cycle=2004 \
113+
"$ID"
114+
115+
# it should have reached the 2004 cycle
116+
grep '2004/plot' "${HOME}/cylc-run/${ID}/log/scheduler/log"
117+
grep '2004/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
118+
! grep '2005/run_model' "${HOME}/cylc-run/${ID}/log/scheduler/log"
119+
120+
# clean up
121+
cylc clean "$ID"
122+
123+
rm -r "${SRC_DIR}"
124+
}
125+
126+
127+
# test_simple
128+
test_complex

‎cylc/flow/etc/examples/extending-workflow/complex/flow.cylc

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!Jinja2
2+
3+
{#
4+
override this default on the command line
5+
e.g. "cylc play -s stop_cycle=2020"
6+
#}
7+
{% set stop_cycle = stop_cycle | default('2010') %}
8+
9+
[scheduler]
10+
# use the year for the cycle point
11+
# (strip off the month, day, hour and minute)
12+
cycle point format = CCYY
13+
allow implicit tasks = True
14+
15+
[scheduling]
16+
initial cycle point = 2000
17+
stop after cycle point = {{ stop_cycle }}
18+
[[graph]]
19+
# first cycle only
20+
R1 = """
21+
build => run_model
22+
install => run_model
23+
"""
24+
25+
# first three cycles only
26+
R3/2000/P1Y = """
27+
initial_conditions => run_model
28+
"""
29+
30+
# every cycle
31+
P1Y = """
32+
run_model[-P1Y] => run_model
33+
"""
34+
35+
# last three cycles
36+
R3/P1Y/{{ stop_cycle }} = """
37+
process[-P1Y] => process
38+
run_model => process
39+
"""
40+
41+
# last cycle
42+
R1/{{ stop_cycle }} = """
43+
process => analyse => plot
44+
"""

‎cylc/flow/etc/examples/extending-workflow/index.rst

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
Extending Workflow
2+
==================
3+
4+
.. cylc:scope:: [scheduling]
5+
6+
Sometimes we may run a workflow to completion, but subsequently wish to
7+
run it for a few more cycles.
8+
9+
With Cylc 7 this was often done by changing the `final cycle point` and
10+
restarting the workflow. This approach worked, but was a little awkward.
11+
It's possible with Cylc 8, but we would recommend moving away from this
12+
pattern instead.
13+
14+
The recommended approach to this problem (Cylc 6+) is to use the
15+
`stop after cycle point` rather than the `final cycle point`.
16+
17+
The `stop after cycle point` tells Cylc to **stop** after the workflow passes
18+
the specified point, whereas the `final cycle point` tells Cylc that the
19+
workflow **finishes** at the specified point.
20+
21+
When a workflow **finishes**, it is a little awkward to restart as you have to
22+
tell Cylc which tasks to continue on from. The `stop after cycle point`
23+
solution avoids this issue.
24+
25+
26+
Simple Example
27+
--------------
28+
29+
.. admonition:: Get a copy of this example
30+
:class: hint
31+
32+
.. code-block:: console
33+
34+
$ cylc get-resources examples/extending-workflow/simple
35+
36+
This workflow will stop at the end of the ``2002`` cycle:
37+
38+
.. literalinclude:: simple/flow.cylc
39+
:language: cylc
40+
41+
After it has run and shut down, change the `stop after cycle point` to
42+
the desired value and restart it. E.g:
43+
44+
.. code-block:: bash
45+
46+
# install and run the workflow:
47+
cylc vip
48+
49+
# then later edit "stop after cycle point" to "2004"
50+
51+
# then reinstall and restart the workflow:
52+
cylc vr
53+
54+
The workflow will continue from where it left off and run until the end of the
55+
``2004`` cycle. Because the workflow never hit the `final cycle point` it
56+
never "finished" so no special steps are required to restart the workflow.
57+
58+
You can also set the `stop after cycle point` when you start the workflow:
59+
60+
.. code-block:: bash
61+
62+
cylc play --stop-cycle-point=2020 myworkflow
63+
64+
Or change it at any point whilst the workflow is running:
65+
66+
.. code-block:: bash
67+
68+
cylc stop myworkflow//2030 # change the stop after cycle point to 2030
69+
70+
.. note::
71+
72+
If you set the `stop after cycle point` on the command line, this value will
73+
take precedence over the one in the workflow configuration. Use
74+
``cylc play --stop-cycle-point=reload`` to restart the workflow using the
75+
`stop after cycle point` configured in the workflow configuration.
76+
77+
78+
Complex Example
79+
---------------
80+
81+
.. admonition:: Get a copy of this example
82+
:class: hint
83+
84+
.. code-block:: console
85+
86+
$ cylc get-resources examples/extending-workflow/complex
87+
88+
Sometimes we may want to run some tasks at the `final cycle point` or in the
89+
cycles leading up to the `final cycle point`. For example you might have some
90+
analysis or housekeeping tasks to run at the end of the workflow.
91+
92+
We can schedule such task like so:
93+
94+
.. code-block:: cylc
95+
96+
# run once in the "2000" cycle
97+
R1/2000 # 2000
98+
99+
# run every year three times ending in 2000
100+
R3/P1Y/2000 # 1998, 1999, 2000
101+
102+
For more information see :ref:`user_guide.cycling_format_4`.
103+
104+
This example uses Jinja2 to template the cycling expressions above to schedule
105+
tasks to run at, and in the run up to, the `stop after cycle point`:
106+
107+
.. literalinclude:: complex/flow.cylc
108+
:language: cylc
109+
110+
.. code-block:: bash
111+
112+
# install and run the workflow:
113+
cylc vip
114+
115+
# then reinstall and restart the workflow extending the stop cycle:
116+
cylc vr -s stop_cycle=2020
117+
118+
119+
.. cylc:scope::

‎cylc/flow/etc/examples/extending-workflow/simple/flow.cylc

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
[scheduler]
2+
# use the year for the cycle point
3+
# (strip off the month, day, hour and minute)
4+
cycle point format = CCYY
5+
6+
[scheduling]
7+
initial cycle point = 2000
8+
stop after cycle point = 2002 # stop after two years of simulated time
9+
[[graph]]
10+
P1Y = """
11+
z[-P1Y] => a
12+
a => z
13+
"""
14+
15+
[runtime]
16+
[[a]]
17+
[[z]]

0 commit comments

Comments
 (0)