Skip to content

Commit

Permalink
fix: address comment
Browse files Browse the repository at this point in the history
  • Loading branch information
dvilelaf committed Jun 5, 2024
1 parent 17b738e commit baf339a
Show file tree
Hide file tree
Showing 2 changed files with 18 additions and 25 deletions.
1 change: 0 additions & 1 deletion operate/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -185,7 +185,6 @@ def schedule_healthcheck_job(
healthcheck_jobs[service] = loop.create_task(
operate.service_manager().healthcheck_job(
hash=service,
loop=loop,
)
)

Expand Down
42 changes: 18 additions & 24 deletions operate/services/manage.py
Original file line number Diff line number Diff line change
Expand Up @@ -869,34 +869,28 @@ async def funding_job(
async def healthcheck_job(
self,
hash: str,
loop: t.Optional[asyncio.AbstractEventLoop] = None,
) -> None:
"""Start a background funding job."""
loop = loop or asyncio.get_event_loop()
failed_health_checks = 0

with ThreadPoolExecutor() as executor:
while True:
try:
# Check the service health
healthy = await loop.run_in_executor(
executor,
check_service_health,
)
# Restart the service if the health failed 5 times in a row
if not healthy:
failed_health_checks += 1
else:
failed_health_checks = 0
if failed_health_checks >= 5:
self.stop_service_locally(hash=hash)
self.deploy_service_locally(hash=hash)

except Exception: # pylint: disable=broad-except
logging.info(
f"Error occured while checking the service health\n{traceback.format_exc()}"
)
await asyncio.sleep(60)
while True:
try:
# Check the service health
healthy = await check_service_health()
# Restart the service if the health failed 5 times in a row
if not healthy:
failed_health_checks += 1
else:
failed_health_checks = 0
if failed_health_checks >= 5:
self.stop_service_locally(hash=hash)
self.deploy_service_locally(hash=hash)

except Exception: # pylint: disable=broad-except
logging.info(
f"Error occured while checking the service health\n{traceback.format_exc()}"
)
await asyncio.sleep(60)

def deploy_service_locally(self, hash: str, force: bool = True) -> Deployment:
"""
Expand Down

0 comments on commit baf339a

Please sign in to comment.