fix htz offline detection

EthDevOps · May 7, 2024 · 81b7b53 · 81b7b53
1 parent 1b1c57a
commit 81b7b53
Showing 1 changed file with 8 additions and 0 deletions.
diff --git a/PoolManager.cs b/PoolManager.cs
@@ -52,6 +52,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
 
             try
             {
+                // update the world state for htz
+                allHtzSrvs = await _cc.GetAllServers();
                 foreach (GithubTargetConfiguration tgt in targetConfig)
                 {
                     GithubRunnersGauge.Labels(tgt.Name, "active").Set(0);
@@ -95,6 +97,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             if (DateTime.UtcNow - crudeTimer > TimeSpan.FromMinutes(cullMinutes))
             {
                 _logger.LogInformation("Cleaning runners...");
+                // update the world state for htz
+                allHtzSrvs = await _cc.GetAllServers();
                 await CleanUpRunners(targetConfig, allHtzSrvs);
                 await StartPoolRunners(targetConfig);
                 crudeTimer = DateTime.UtcNow;
@@ -207,6 +211,10 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs,
                     // VM younger than 30min - not cleaning yet
                     continue;
                 }
+                if (htzSrv == null)
+                {
+                    continue;
+                }
 
                 _logger.LogInformation($"Removing offline runner {runnerToRemove.Name} from org {githubTarget.Name}");
                 bool _ = githubTarget.Target switch