From 9c8b0a1fc8c5bb4cfeec029cd90e1243735a0f7a Mon Sep 17 00:00:00 2001 From: Markus Keil Date: Tue, 23 Apr 2024 08:23:24 +0200 Subject: [PATCH] add gh registration stats and reduce cull time to 10min --- PoolManager.cs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/PoolManager.cs b/PoolManager.cs index df77aee..3e6e7aa 100644 --- a/PoolManager.cs +++ b/PoolManager.cs @@ -26,6 +26,8 @@ public class PoolManager : BackgroundService .CreateCounter("github_machines_created", "Number of created machines", labelNames: ["org","size"]); private static readonly Gauge QueueSize = Metrics .CreateGauge("github_queue", "Number of queued runner tasks"); + private static readonly Gauge GithubRunnersGauge = Metrics + .CreateGauge("github_registered_runners", "Number of runners registered to github actions", labelNames: ["org", "status"]); private readonly RunnerQueue _queues; @@ -56,10 +58,26 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Yield(); DateTime crudeTimer = DateTime.UtcNow; - int cullMinutes = 30; + int cullMinutes = 10; while (!stoppingToken.IsCancellationRequested) { + // Grab some stats + QueueSize.Set(_queues.CreateTasks.Count + _queues.DeleteTasks.Count); + + foreach (var org in orgConfig) + { + var orgRunners = await GitHubApi.GetRunners(org.GitHubToken, org.OrgName); + var ghStatus = orgRunners.runners.GroupBy(x => x.status).Select(x => new {Status = x.Key, Count = x.Count()}); + foreach (var ghs in ghStatus) + { + GithubRunnersGauge.Labels(org.OrgName, ghs.Status).Set(ghs.Count); + _logger.LogInformation($"GitHub register stats for {org.OrgName}: {ghs.Status}={ghs.Count}"); + } + + } + + // check for culling interval if (DateTime.UtcNow - crudeTimer > TimeSpan.FromMinutes(cullMinutes)) { @@ -68,8 +86,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await StartPoolRunners(orgConfig); crudeTimer = DateTime.UtcNow; } - - QueueSize.Set(_queues.CreateTasks.Count + _queues.DeleteTasks.Count); + if (_queues.DeleteTasks.TryDequeue(out DeleteRunnerTask? dtask)) {