Skip to content

Commit

Permalink
Fix bug related to how unhealthy runners are removed
Browse files Browse the repository at this point in the history
  • Loading branch information
yhaliaw committed Jul 17, 2023
1 parent bb94609 commit ce84c24
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 9 deletions.
5 changes: 4 additions & 1 deletion src/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def remove(self, remove_token: str) -> None:
Raises:
RunnerRemoveError: Failure in removing runner.
"""
logger.info("Removing LXD instance of runner: %s", self.config.name)
logger.info("Removing runner: %s", self.config.name)

if self.instance:
logger.info("Executing command to removal of runner and clean up...")
Expand All @@ -153,6 +153,7 @@ def remove(self, remove_token: str) -> None:
)

if self.instance.status == "Running":
logger.info("Removing LXD instance of runner: %s", self.config.name)
try:
self.instance.stop(wait=True, timeout=60)
except LxdError:
Expand Down Expand Up @@ -180,6 +181,8 @@ def remove(self, remove_token: str) -> None:
if self.status.runner_id is None:
return

logger.info("Removing runner on GitHub: %s", self.config.name)

# The runner should cleanup itself. Cleanup on GitHub in case of runner cleanup error.
if isinstance(self.config.path, GitHubRepo):
logger.debug(
Expand Down
10 changes: 2 additions & 8 deletions src/runner_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -303,16 +303,10 @@ def reconcile(self, quantity: int, resources: VirtualMachineResources) -> int:
logger.info("Cleaning up unhealthy runners.")

remove_token = self._get_github_remove_token()
runners = {runner.config.name: runner for runner in self._get_runners()}

for instance in runner_lxd_instances.unhealthy:
config = RunnerConfig(
self.app_name,
self.config.path,
self.proxies,
self.config.lxd_storage_path,
instance.name,
)
runner = Runner(self._clients, config, RunnerStatus())
runner = runners[instance.name]
runner.remove(remove_token)
logger.info("Removed runner: %s", runner.config.name)

Expand Down

0 comments on commit ce84c24

Please sign in to comment.