diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index e276f077efb..169c5eb66ac 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -220,16 +220,13 @@ def get_script_common_text(this: str, example: Optional[str] = None): The following directories are installed by default: - * app - * bin - * etc - * lib - - And include the server.key file (from the .service - directory), this is required for authentication. + * ``app/`` + * ``bin/`` + * ``etc/`` + * ``lib/`` These should be located in the top level of your Cylc workflow, - i.e. the directory that contains your flow.cylc file. + i.e. the directory that contains your ``flow.cylc`` file. Directories must have a trailing slash. For example, to add the following items to your file installation: @@ -1862,8 +1859,9 @@ def warn_about_depr_platform(cfg): if depr: msg = "\n".join(depr) LOG.warning( - f'Task {task_name}: deprecated "host" and "batch system"' - f' use "platform".\n{msg}' + "deprecated settings found " + f"(please replace with [runtime][{task_name}]platform):" + f"\n{msg}" ) diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 5a76b89dcd9..c02c204b8dd 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -164,13 +164,6 @@ class WorkflowConfig: CHECK_CIRCULAR_LIMIT = 100 # If no. tasks > this, don't check circular VIS_N_POINTS = 3 - CYLC7_GRAPH_COMPAT_MSG = ( - "Cylc 7 graph compatibility: making success outputs 'required' (to" - " retain failed tasks in the pool) and pre-spawning graph children (to" - " replicate Cylc 7 stall behaviour). Please refer to documentation on" - " upgrading Cylc 7 graphs to Cylc 8." - ) - def __init__( self, workflow: str, @@ -629,7 +622,8 @@ def process_initial_cycle_point(self) -> None: except IsodatetimeError as exc: raise WorkflowConfigError(str(exc)) if orig_icp != icp: - # now/next()/prev() was used, need to store evaluated point in DB + # now/next()/previous() was used, need to store + # evaluated point in DB self.options.icp = icp self.initial_point = get_point(icp).standardise() self.cfg['scheduling']['initial cycle point'] = str(self.initial_point) @@ -802,20 +796,21 @@ def _check_implicit_tasks(self) -> None: raise WorkflowConfigError(msg) # Otherwise "[scheduler]allow implicit tasks" is not set - msg = ( + + # Allow implicit tasks in back-compat mode (unless rose-suite.conf + # present, to maintain compat with Rose 2019) + if ( + cylc.flow.flags.cylc7_back_compat and + not Path(self.run_dir, 'rose-suite.conf').is_file() + ): + LOG.debug(msg) + return + + raise WorkflowConfigError( f"{msg}\n" "To allow implicit tasks, use " - f"'{WorkflowFiles.FLOW_FILE}[scheduler]allow implicit tasks'\n" - "See https://cylc.github.io/cylc-doc/latest/html/" - "7-to-8/summary.html#backward-compatibility" + f"'{WorkflowFiles.FLOW_FILE}[scheduler]allow implicit tasks'" ) - # Allow implicit tasks in Cylc 7 back-compat mode (but not if - # rose-suite.conf present, to maintain compat with Rose 2019) - if ( - Path(self.run_dir, 'rose-suite.conf').is_file() or - not cylc.flow.flags.cylc7_back_compat - ): - raise WorkflowConfigError(msg) def _check_circular(self): """Check for circular dependence in graph.""" @@ -2010,8 +2005,6 @@ def load_graph(self): sections.append((section, value)) # Parse and process each graph section. - if cylc.flow.flags.cylc7_back_compat: - LOG.warning(self.__class__.CYLC7_GRAPH_COMPAT_MSG) task_triggers = {} task_output_opt = {} for section, graph in sections: diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py index 339d0d87c3e..da18a9bb07c 100644 --- a/cylc/flow/cycling/iso8601.py +++ b/cylc/flow/cycling/iso8601.py @@ -617,11 +617,11 @@ def _get_old_anchor_step_recurrence(anchor, step, start_point): return str(anchor_point) + "/" + str(step) -def ingest_time(value: str, now: Optional['TimePoint'] = None) -> str: +def ingest_time(value: str, now: Optional[str] = None) -> str: """Handle relative, truncated and prev/next cycle points. Args: - value: The string containing the prev()/next() stuff. + value: The string containing the previous()/next() stuff. now: A time point to use as the context for resolving the value. """ # remove extraneous whitespace from cycle point @@ -635,7 +635,7 @@ def ingest_time(value: str, now: Optional['TimePoint'] = None) -> str: (value.startswith("-") or value.startswith("+")) and "P" not in value ) - # prev() or next() + # previous() or next() is_prev_next = "next" in value or "previous" in value # offset from now (±P...) is_offset = value.startswith("P") or value.startswith("-P") @@ -657,34 +657,30 @@ def ingest_time(value: str, now: Optional['TimePoint'] = None) -> str: # missing date-time components off the front (e.g. 01T00) is_truncated = timepoint.truncated - if not any((is_prev_next, is_offset, is_truncated)): + if not (is_prev_next or is_offset or is_truncated): return value if now is None: - now = parser.parse(get_current_time_string()) - else: - now = parser.parse(now) + now = get_current_time_string() + now_point = parser.parse(now) - # correct for year in 'now' if year only, - # or year and time, specified in input - # TODO: Figure out why this correction is needed + # correct for year in 'now' if year is the only date unit specified - + # https://github.com/cylc/cylc-flow/issues/4805#issuecomment-1103928604 if re.search(r"\(-\d{2}[);T]", value): - now += Duration(years=1) - - # correct for month in 'now' if year and month only, - # or year, month and time, specified in input + now_point += Duration(years=1) + # likewise correct for month if year and month are the only date units elif re.search(r"\(-\d{4}[);T]", value): - now += Duration(months=1) + now_point += Duration(months=1) # perform whatever transformation is required offset = None if is_prev_next: - cycle_point, offset = prev_next(value, now, parser) + cycle_point, offset = prev_next(value, now_point, parser) elif is_offset: - cycle_point = now + cycle_point = now_point offset = value else: # is_truncated - cycle_point = now + timepoint + cycle_point = now_point + timepoint if offset is not None: # add/subtract offset duration to/from chosen timepoint @@ -700,10 +696,10 @@ def ingest_time(value: str, now: Optional['TimePoint'] = None) -> str: def prev_next( value: str, now: 'TimePoint', parser: 'TimePointParser' ) -> Tuple['TimePoint', Optional[str]]: - """Handle prev() and next() syntax. + """Handle previous() and next() syntax. Args: - value: The string containing the prev()/next() stuff. + value: The string containing the previous()/next() stuff. now: A time point to use as the context for resolving the value. parser: A time point parser. @@ -713,7 +709,7 @@ def prev_next( # are we in gregorian mode (or some other eccentric calendar if CALENDAR.mode != Calendar.MODE_GREGORIAN: raise CylcConfigError( - 'prev()/next() syntax must be used with integer or gregorian' + 'previous()/next() syntax must be used with integer or gregorian' f' cycling modes ("{value}")' ) @@ -759,8 +755,8 @@ def prev_next( cycle_point = timepoints[my_diff.index(min(my_diff))] - # ensure truncated dates do not have - # time from 'now' included' + # ensure truncated dates do not have time from 'now' included' - + # https://github.com/metomi/isodatetime/issues/212 if 'T' not in value.split(')')[0]: # NOTE: Strictly speaking we shouldn't forcefully mutate TimePoints # in this way as they're meant to be immutable since @@ -771,18 +767,14 @@ def prev_next( cycle_point._hour_of_day = 0 cycle_point._minute_of_hour = 0 cycle_point._second_of_minute = 0 - - # ensure month and day from 'now' are not included + # likewise ensure month and day from 'now' are not included # where they did not appear in the truncated datetime - # NOTE: this may break when the order of tick over - # for time point is reversed!!! - # https://github.com/metomi/isodatetime/pull/101 - # case 1 - year only if re.search(r"\(-\d{2}[);T]", value): + # case 1 - year only cycle_point._month_of_year = 1 cycle_point._day_of_month = 1 - # case 2 - month only or year and month elif re.search(r"\(-(-\d{2}|\d{4})[;T)]", value): + # case 2 - month only or year and month cycle_point._day_of_month = 1 return cycle_point, offset diff --git a/cylc/flow/workflow_files.py b/cylc/flow/workflow_files.py index 3d8c35a3e98..de126b7cadf 100644 --- a/cylc/flow/workflow_files.py +++ b/cylc/flow/workflow_files.py @@ -338,24 +338,13 @@ class RemoteCleanQueueTuple(NamedTuple): * ssh -n "%(host)s" kill %(pid)s # final brute force! """ -SUITERC_DEPR_MSG = ( - "Backward compatibility mode ON for CYLC 7" - f" '{WorkflowFiles.SUITE_RC}' config files." - " When ready to upgrade, rename the file to" - f" {WorkflowFiles.FLOW_FILE} then address " - "any resulting validation errors and warnings." -) +SUITERC_DEPR_MSG = "Backward compatibility mode ON" NO_FLOW_FILE_MSG = ( f"No {WorkflowFiles.FLOW_FILE} or {WorkflowFiles.SUITE_RC} " "in {}" ) -REG_CLASH_MSG = ( - "The specified reg could refer to ./{0} or ~/cylc-run/{1}. " - "This command will use ./{0}." -) - NESTED_DIRS_MSG = ( "Nested {dir_type} directories not allowed - cannot install workflow" " in '{dest}' as '{existing}' is already a valid {dir_type} directory." diff --git a/tests/functional/cylc-clean/02-targeted.t b/tests/functional/cylc-clean/02-targeted.t index 317b86c68d4..1b4c319332f 100644 --- a/tests/functional/cylc-clean/02-targeted.t +++ b/tests/functional/cylc-clean/02-targeted.t @@ -35,7 +35,7 @@ create_test_global_config "" " log = ${TEST_DIR}/${SYM_NAME}/other share = ${TEST_DIR}/${SYM_NAME}/other work = ${TEST_DIR}/${SYM_NAME}/other - # Need to override any symlink dirs set in global-tests.cylc: + # Need to override any symlink dirs set in global.cylc: share/cycle = " install_workflow "${TEST_NAME_BASE}" basic-workflow diff --git a/tests/functional/cylc-install/03-file-transfer.t b/tests/functional/cylc-install/03-file-transfer.t index 50800eeed82..e35b527e2b6 100644 --- a/tests/functional/cylc-install/03-file-transfer.t +++ b/tests/functional/cylc-install/03-file-transfer.t @@ -23,6 +23,18 @@ if ! command -v 'tree' >'/dev/null'; then fi set_test_number 6 +# Need to override any symlink dirs set in global.cylc: +create_test_global_config "" " +[install] + [[symlink dirs]] + [[[localhost]]] + run = + log = + work = + share = + share/cycle = +" + # Test cylc install copies files to run dir successfully. TEST_NAME="${TEST_NAME_BASE}-basic" make_rnd_workflow diff --git a/tests/functional/cylc-reinstall/01-file-transfer.t b/tests/functional/cylc-reinstall/01-file-transfer.t index 5e3e4366716..b96e5d49198 100644 --- a/tests/functional/cylc-reinstall/01-file-transfer.t +++ b/tests/functional/cylc-reinstall/01-file-transfer.t @@ -23,6 +23,18 @@ if ! command -v 'tree' >'/dev/null'; then fi set_test_number 9 +# Need to override any symlink dirs set in global.cylc: +create_test_global_config "" " +[install] + [[symlink dirs]] + [[[localhost]]] + run = + log = + work = + share = + share/cycle = +" + # Test cylc install copies files to run dir successfully. TEST_NAME="${TEST_NAME_BASE}-basic" make_rnd_workflow diff --git a/tests/functional/optional-outputs/03-c7backcompat.t b/tests/functional/optional-outputs/03-c7backcompat.t index 432430e09bf..cc868ac2017 100644 --- a/tests/functional/optional-outputs/03-c7backcompat.t +++ b/tests/functional/optional-outputs/03-c7backcompat.t @@ -23,7 +23,7 @@ # sufficient to check the resulting validation and run time behaviour. . "$(dirname "$0")/test_header" -set_test_number 6 +set_test_number 5 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" @@ -44,10 +44,6 @@ DEPR_MSG_1=$(python -c \ 'from cylc.flow.workflow_files import SUITERC_DEPR_MSG; print(SUITERC_DEPR_MSG)') grep_ok "${DEPR_MSG_1}" "${TEST_NAME}.stderr" -DEPR_MSG_2=$(python -c \ - 'from cylc.flow.config import WorkflowConfig as cfg; print(cfg.CYLC7_GRAPH_COMPAT_MSG);') -grep_ok "${DEPR_MSG_2}" "${TEST_NAME}.stderr" - # And it should run without stalling with an incomplete task. workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play -n --reference-test --debug "${WORKFLOW_NAME}" diff --git a/tests/functional/optional-outputs/04-c7backcompat-blocked-task.t b/tests/functional/optional-outputs/04-c7backcompat-blocked-task.t index d62f83ce476..c15175caa63 100644 --- a/tests/functional/optional-outputs/04-c7backcompat-blocked-task.t +++ b/tests/functional/optional-outputs/04-c7backcompat-blocked-task.t @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Cylc 7 stall backward compatibility, single parent case +# Cylc 7 stall backward compatibility, single parent case . "$(dirname "$0")/test_header" -set_test_number 8 +set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" @@ -30,10 +30,6 @@ DEPR_MSG_1=$(python -c \ 'from cylc.flow.workflow_files import SUITERC_DEPR_MSG; print(SUITERC_DEPR_MSG)') grep_ok "${DEPR_MSG_1}" "${TEST_NAME}.stderr" -DEPR_MSG_2=$(python -c \ - 'from cylc.flow.config import WorkflowConfig as cfg; print(cfg.CYLC7_GRAPH_COMPAT_MSG);') -grep_ok "${DEPR_MSG_2}" "${TEST_NAME}.stderr" - # Should stall and abort with an unsatisfied prerequisite. workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play -n --reference-test --debug "${WORKFLOW_NAME}" diff --git a/tests/functional/optional-outputs/05-c7backcompat-blocked-task-2.t b/tests/functional/optional-outputs/05-c7backcompat-blocked-task-2.t index a914d6504e0..e08be9a0a69 100644 --- a/tests/functional/optional-outputs/05-c7backcompat-blocked-task-2.t +++ b/tests/functional/optional-outputs/05-c7backcompat-blocked-task-2.t @@ -15,10 +15,10 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -# Cylc 7 stall backward compatibility, multi-parent case +# Cylc 7 stall backward compatibility, multi-parent case . "$(dirname "$0")/test_header" -set_test_number 8 +set_test_number 7 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" @@ -30,10 +30,6 @@ DEPR_MSG_1=$(python -c \ 'from cylc.flow.workflow_files import SUITERC_DEPR_MSG; print(SUITERC_DEPR_MSG)') grep_ok "${DEPR_MSG_1}" "${TEST_NAME}.stderr" -DEPR_MSG_2=$(python -c \ - 'from cylc.flow.config import WorkflowConfig as cfg; print(cfg.CYLC7_GRAPH_COMPAT_MSG);') -grep_ok "${DEPR_MSG_2}" "${TEST_NAME}.stderr" - # Should stall and abort with an unsatisfied prerequisite. workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play -n --reference-test --debug "${WORKFLOW_NAME}" diff --git a/tests/functional/optional-outputs/06-c7backcompat-family.t b/tests/functional/optional-outputs/06-c7backcompat-family.t index 026917b75f7..0593f4d5b83 100644 --- a/tests/functional/optional-outputs/06-c7backcompat-family.t +++ b/tests/functional/optional-outputs/06-c7backcompat-family.t @@ -18,7 +18,7 @@ # Cylc 7 stall backward compatibility, complex family case. . "$(dirname "$0")/test_header" -set_test_number 13 +set_test_number 12 install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" @@ -30,10 +30,6 @@ DEPR_MSG_1=$(python -c \ 'from cylc.flow.workflow_files import SUITERC_DEPR_MSG; print(SUITERC_DEPR_MSG)') grep_ok "${DEPR_MSG_1}" "${TEST_NAME}.stderr" -DEPR_MSG_2=$(python -c \ - 'from cylc.flow.config import WorkflowConfig as cfg; print(cfg.CYLC7_GRAPH_COMPAT_MSG);') -grep_ok "${DEPR_MSG_2}" "${TEST_NAME}.stderr" - # Should stall and abort with unsatisfied "stall" tasks. workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play -n --debug "${WORKFLOW_NAME}" diff --git a/tests/functional/validate/37-special-implicit-task.t b/tests/functional/validate/37-special-implicit-task.t index 179e7fc56fb..b9845e9154f 100755 --- a/tests/functional/validate/37-special-implicit-task.t +++ b/tests/functional/validate/37-special-implicit-task.t @@ -34,6 +34,5 @@ cmp_ok "${TEST_NAME_BASE}.stderr" << '__ERR__' WorkflowConfigError: implicit tasks detected (no entry under [runtime]): * foo To allow implicit tasks, use 'flow.cylc[scheduler]allow implicit tasks' -See https://cylc.github.io/cylc-doc/latest/html/7-to-8/summary.html#backward-compatibility __ERR__ exit diff --git a/tests/functional/validate/67-relative-icp.t b/tests/functional/validate/67-relative-icp.t index f9c9d5d70e3..05a678f1acf 100755 --- a/tests/functional/validate/67-relative-icp.t +++ b/tests/functional/validate/67-relative-icp.t @@ -16,25 +16,25 @@ # along with this program. If not, see . # Checking that syntax of relative initial cycle point is validated. -# NOTE: THIS TEST WILL FAIL FROM 01/01/2117 +# Note: remember to update this test after 01/01/2117 . "$(dirname "$0")/test_header" - -set_test_number 2 +set_test_number 3 cat >'flow.cylc' <<'__FLOW_CONFIG__' [scheduler] UTC mode = true + allow implicit tasks = True [scheduling] initial cycle point = previous(-17T1200Z; -18T1200Z) - P1D [[graph]] P1D = t1 -[runtime] - [[t1]] - script = true __FLOW_CONFIG__ -run_ok "${TEST_NAME_BASE}" cylc graph --reference . -grep_ok "20171231T1200Z/t1" "${TEST_NAME_BASE}.stdout" +run_ok "${TEST_NAME_BASE}-val" cylc validate . + +TEST_NAME="${TEST_NAME_BASE}-graph" +run_ok "$TEST_NAME" cylc graph --reference . +grep_ok "20171231T1200Z/t1" "${TEST_NAME}.stdout" exit diff --git a/tests/unit/cfgspec/test_workflow.py b/tests/unit/cfgspec/test_workflow.py index 1a8e44df90c..7adbd319d83 100644 --- a/tests/unit/cfgspec/test_workflow.py +++ b/tests/unit/cfgspec/test_workflow.py @@ -66,7 +66,7 @@ 'job': {'batch system': 'pbs'} } }, - False, "Task bar: deprecated \"host\" and \"batch system\"", + False, "please replace with [runtime][bar]platform", id="Deprecated settings" ) ] diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index d88e181e6cb..ce3baac3054 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1134,16 +1134,14 @@ def test_invalid_custom_output_msg(tmp_flow_config: Callable): ) in str(cm.value) -def test_c7_back_compat_optional_outputs(tmp_flow_config, monkeypatch, caplog): +def test_c7_back_compat_optional_outputs(tmp_flow_config, monkeypatch): """Test optional and required outputs Cylc 7 back compat mode. Success outputs should be required, others optional. Tested here because success is set to required after graph parsing, in taskdef processing. """ - caplog.set_level(logging.WARNING, CYLC_LOG) - monkeypatch.setattr( - 'cylc.flow.flags.cylc7_back_compat', True) + monkeypatch.setattr('cylc.flow.flags.cylc7_back_compat', True) reg = 'custom_out2' flow_file = tmp_flow_config(reg, ''' [scheduling] @@ -1161,7 +1159,6 @@ def test_c7_back_compat_optional_outputs(tmp_flow_config, monkeypatch, caplog): ''') cfg = WorkflowConfig(workflow=reg, fpath=flow_file, options=None) - assert WorkflowConfig.CYLC7_GRAPH_COMPAT_MSG in caplog.text for taskdef in cfg.taskdefs.values(): for output, (_, required) in taskdef.outputs.items(): @@ -1227,22 +1224,22 @@ def test_success_after_optional_submit(tmp_flow_config, graph): ] ) @pytest.mark.parametrize( - 'cylc7_compat, rose_suite_conf, expected_exc', + 'cylc7_compat, rose_suite_conf, expected_exc, extra_msg', [ pytest.param( - False, False, WorkflowConfigError, + False, False, WorkflowConfigError, True, id="Default" ), pytest.param( - False, True, WorkflowConfigError, + False, True, WorkflowConfigError, True, id="rose-suite.conf present" ), pytest.param( - True, False, None, + True, False, None, False, id="Cylc 7 back-compat" ), pytest.param( - True, True, WorkflowConfigError, + True, True, WorkflowConfigError, False, id="Cylc 7 back-compat, rose-suite.conf present" ), ] @@ -1252,6 +1249,7 @@ def test_implicit_tasks( cylc7_compat: bool, rose_suite_conf: bool, expected_exc: Optional[Type[Exception]], + extra_msg: bool, caplog: pytest.LogCaptureFixture, log_filter: Callable, monkeypatch: pytest.MonkeyPatch, @@ -1266,6 +1264,8 @@ def test_implicit_tasks( rose_suite_conf: Whether a rose-suite.conf file is present in run dir. expected_exc: Exception expected to be raised only when "[scheduler]allow implicit tasks" is not set. + extra_msg: If True, there should be the note on how to allow implicit + tasks in the err msg. """ # Setup reg = 'rincewind' @@ -1289,11 +1289,12 @@ def test_implicit_tasks( expected_exc = WorkflowConfigError # Test args: dict = {'workflow': reg, 'fpath': flow_file, 'options': None} - expected_msg = "implicit tasks detected" + expected_msg = r"implicit tasks detected.*" + if allow_implicit_tasks is not False and extra_msg: + expected_msg += r"[\s\S]*To allow implicit tasks.*" if expected_exc: - with pytest.raises(expected_exc) as exc: + with pytest.raises(expected_exc, match=expected_msg): WorkflowConfig(**args) - assert expected_msg in str(exc.value) else: WorkflowConfig(**args)