Skip to content

Commit

Permalink
fileinstall: run coros in the background if loop already running
Browse files Browse the repository at this point in the history
* Allow Rose file installation to be called by code which already
  has an event loop running by scheduling coroutines to run in the
  background (i.e. schedule but don't await).
* The calling code can list these tasks using `asyncio.all_tasks()`
  and await them as appropriate.
* Addresses cylc/cylc-rose#274
  • Loading branch information
oliver-sanders committed Dec 6, 2023
1 parent 56dc066 commit 7136d7c
Showing 1 changed file with 16 additions and 7 deletions.
23 changes: 16 additions & 7 deletions metomi/rose/job_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,19 +175,28 @@ def run(self, job_manager, *args, concurrency=6):
The maximum number of jobs to run concurrently.
"""
running = []
loop = asyncio.get_event_loop()
loop.set_exception_handler(self.job_processor.handle_event)
loop.run_until_complete(
asyncio.gather(
self._run_jobs(running, job_manager, args, concurrency),
self._post_process_jobs(running, job_manager, args),
)
)
coro = self._run(job_manager, *args, concurrency=concurrency)
try:
# event loop is not running (e.g. rose CLI use)
loop.run_until_complete(coro)
except RuntimeError:
# event loop is already running (e.g. cylc CLI use)
# WARNING: this starts the file installation running, but it
# doesn't wait for it to finish, that's your problem :(
loop.create_task(coro)
dead_jobs = job_manager.get_dead_jobs()
if dead_jobs:
raise JobRunnerNotCompletedError(dead_jobs)

async def _run(self, job_manager, *args, concurrency=6):
running = []
asyncio.gather(
self._run_jobs(running, job_manager, args, concurrency),
self._post_process_jobs(running, job_manager, args),
)

async def _run_jobs(self, running, job_manager, args, concurrency):
"""Run pending jobs subject to the concurrency limit.
Expand Down

0 comments on commit 7136d7c

Please sign in to comment.