From c14041289fdb0708366851d8c6b4817724b25a89 Mon Sep 17 00:00:00 2001 From: Maxime RIO Date: Wed, 24 Apr 2024 10:57:11 +1200 Subject: [PATCH 1/6] fix bug when job runner submit stdout doesn't end with newline When the stdout returned by the job runner submit method doesn't end with a newline, the `sys.stdout.write` call in `jobs_submit` do not add it and this cases submission to fails (at least to be recorded as failed by the calling process when running a workflow). Looking at how newlines are handled in the JobRunnerManager.job_kill method, it looks like this a typo in the jobs_submit method. --- cylc/flow/job_runner_mgr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cylc/flow/job_runner_mgr.py b/cylc/flow/job_runner_mgr.py index f23c6c70104..ba367e753be 100644 --- a/cylc/flow/job_runner_mgr.py +++ b/cylc/flow/job_runner_mgr.py @@ -306,8 +306,8 @@ def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False, if value is None or not value.strip(): continue for line in value.splitlines(True): - if not value.endswith("\n"): - value += "\n" + if not line.endswith("\n"): + line += "\n" sys.stdout.write("%s%s|%s|[%s] %s" % ( self.OUT_PREFIX_COMMAND, now, job_log_dir, key, line)) From a00fcfac4845371cb3802ca489ad994e50c8496e Mon Sep 17 00:00:00 2001 From: Maxime RIO Date: Wed, 24 Apr 2024 11:02:56 +1200 Subject: [PATCH 2/6] add name to contributors --- CONTRIBUTING.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a4e77fd82a7..fb0b362e4a5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,7 @@ requests_). - Diquan Jabbour - Shixian Sheng - Utheri Wagura + - Maxime Rio (All contributors are identifiable with email addresses in the git version From adbcdfcf5be3b5d65ec9bfad7cc3dabb1fc3828a Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 23 Oct 2024 13:46:46 +1300 Subject: [PATCH 3/6] Tweak as per code review. --- cylc/flow/job_runner_handlers/background.py | 2 +- cylc/flow/job_runner_handlers/documentation.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cylc/flow/job_runner_handlers/background.py b/cylc/flow/job_runner_handlers/background.py index 2866a839dfb..729b9b74d28 100644 --- a/cylc/flow/job_runner_handlers/background.py +++ b/cylc/flow/job_runner_handlers/background.py @@ -89,7 +89,7 @@ def submit(cls, job_file_path, submit_opts): exc.filename = "nohup" return (1, None, str(exc)) else: - return (0, "%d\n" % (proc.pid), None) + return (0, str(proc.pid), None) JOB_RUNNER_HANDLER = BgCommandHandler() diff --git a/cylc/flow/job_runner_handlers/documentation.py b/cylc/flow/job_runner_handlers/documentation.py index 3b546ab4209..415f031684e 100644 --- a/cylc/flow/job_runner_handlers/documentation.py +++ b/cylc/flow/job_runner_handlers/documentation.py @@ -424,8 +424,7 @@ def submit( ret_code: Subprocess return code. out: - Subprocess standard output, note this should be newline - terminated. + Subprocess standard output. err: Subprocess standard error. From 2a43d247424dc2fed1a8f8facfb1943bb7604492 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 23 Oct 2024 14:29:35 +1300 Subject: [PATCH 4/6] Simplify some job_runner code. --- cylc/flow/job_runner_mgr.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cylc/flow/job_runner_mgr.py b/cylc/flow/job_runner_mgr.py index ba367e753be..b8ddaaa0538 100644 --- a/cylc/flow/job_runner_mgr.py +++ b/cylc/flow/job_runner_mgr.py @@ -210,12 +210,10 @@ def jobs_kill(self, job_log_root, job_log_dirs): self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code)) # Note: Print STDERR to STDOUT may look a bit strange, but it # requires less logic for the workflow to parse the output. - if err.strip(): - for line in err.splitlines(True): - if not line.endswith("\n"): - line += "\n" - sys.stdout.write("%s%s|%s|%s" % ( - self.OUT_PREFIX_CMD_ERR, now, job_log_dir, line)) + for line in err.strip().splitlines(): + sys.stdout.write( + f"{self.OUT_PREFIX_CMD_ERR}{now}|{job_log_dir}|{line}\n" + ) def jobs_poll(self, job_log_root, job_log_dirs): """Poll multiple jobs. @@ -303,13 +301,13 @@ def jobs_submit(self, job_log_root, job_log_dirs, remote_mode=False, sys.stdout.write("%s%s|%s|%d|%s\n" % ( self.OUT_PREFIX_SUMMARY, now, job_log_dir, ret_code, job_id)) for key, value in [("STDERR", err), ("STDOUT", out)]: - if value is None or not value.strip(): + if value is None: continue - for line in value.splitlines(True): - if not line.endswith("\n"): - line += "\n" - sys.stdout.write("%s%s|%s|[%s] %s" % ( - self.OUT_PREFIX_COMMAND, now, job_log_dir, key, line)) + for line in value.strip().splitlines(): + sys.stdout.write( + f"{self.OUT_PREFIX_COMMAND}{now}" + f"|{job_log_dir}|[{key}] {line}\n" + ) def job_kill(self, st_file_path): """Ask job runner to terminate the job specified in "st_file_path". From 32aed6af599df963ee01e8cd0500ec6bb3617605 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 23 Oct 2024 14:42:07 +1300 Subject: [PATCH 5/6] Update change log. --- changes.d/6081.fix.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes.d/6081.fix.md diff --git a/changes.d/6081.fix.md b/changes.d/6081.fix.md new file mode 100644 index 00000000000..714321a9311 --- /dev/null +++ b/changes.d/6081.fix.md @@ -0,0 +1,2 @@ +Fix job submission when a batch of jobs is submitted to a runner that does +not return a newline with the job ID (did not affect built-in job runners). From f1fb504801f962a6dc83cae28b9c86522eba4b92 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Wed, 23 Oct 2024 14:48:14 +1300 Subject: [PATCH 6/6] Update mailmap. --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 71a84fdefc6..7bbf47d2375 100644 --- a/.mailmap +++ b/.mailmap @@ -57,3 +57,4 @@ Utheri Wagura <36386988+uwagura@users.noreply.github.com> github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> github-actions[bot] GitHub Action Diquan Jabbour <165976689+Diquan-BOM@users.noreply.github.com> +Maxime Rio