From b7389279eedd1f4972082bc68fc9c286746d4603 Mon Sep 17 00:00:00 2001 From: Markus Keil Date: Mon, 29 Jul 2024 13:52:59 +0200 Subject: [PATCH] add pagination support for githubapi --- GitHubApi.cs | 80 +++++++++++++++++++++++++++++++++----------------- PoolManager.cs | 12 ++++---- 2 files changed, 59 insertions(+), 33 deletions(-) diff --git a/GitHubApi.cs b/GitHubApi.cs index f2e367b..b03cbbd 100644 --- a/GitHubApi.cs +++ b/GitHubApi.cs @@ -8,7 +8,17 @@ namespace GithubActionsOrchestrator; public static class GitHubApi { - public static async Task GetRunnersForOrg(string githubToken, string orgName) + public static async Task> GetRunnersForOrg(string githubToken, string orgName) + { + return await GetRunners(githubToken, $"orgs/{orgName}"); + } + + public static async Task> GetRunnersForRepo(string githubToken, string repoName) + { + return await GetRunners(githubToken, $"repos/{repoName}"); + } + + private static async Task> GetRunners(string githubToken, string ownerPath) { // Register a runner with github @@ -18,42 +28,58 @@ public static async Task GetRunnersForOrg(string githubToken, str client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"Bearer {githubToken}"); client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("hetzner-autoscale", "1")); - HttpResponseMessage response = await client.GetAsync( - $"https://api.github.com/orgs/{orgName}/actions/runners"); - if(response.IsSuccessStatusCode) + var runners = new List(); + string url = $"https://api.github.com/{ownerPath}/actions/runners?per_page=100"; + + while (!string.IsNullOrEmpty(url)) { - string content = await response.Content.ReadAsStringAsync(); - GitHubRunners responseObject = JsonSerializer.Deserialize(content); - return responseObject; + HttpResponseMessage response = await client.GetAsync(url); + if (response.IsSuccessStatusCode) + { + string content = await response.Content.ReadAsStringAsync(); + var pageRunners = JsonSerializer.Deserialize(content); + if (pageRunners != null) + { + runners.AddRange(pageRunners.Runners); + } + + url = GetNextPageUrl(response); + } + else + { + Log.Warning($"Unable to get GH runners for org: [{response.StatusCode}] {response.ReasonPhrase}"); + } } - - Log.Warning($"Unable to get GH runners for org: [{response.StatusCode}] {response.ReasonPhrase}"); - return null; + return runners; } - public static async Task GetRunnersForRepo(string githubToken, string repoName) + private static string GetNextPageUrl(HttpResponseMessage response) { - - // Register a runner with github - using HttpClient client = new(); - client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json")); - client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28"); - client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"Bearer {githubToken}"); - client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("hetzner-autoscale", "1")); - - HttpResponseMessage response = await client.GetAsync( - $"https://api.github.com/repos/{repoName}/actions/runners"); - - if(response.IsSuccessStatusCode) + if (response.Headers.TryGetValues("Link", out var links)) { - string content = await response.Content.ReadAsStringAsync(); - GitHubRunners responseObject = JsonSerializer.Deserialize(content); - return responseObject; + foreach (var link in links) + { + // Example Link header format: ; rel="next", ... + foreach (var part in link.Split(',')) + { + if (part.Contains("rel=\"next\"")) + { + int startIndex = part.IndexOf('<') + 1; + int endIndex = part.IndexOf('>'); + if (startIndex >= 0 && endIndex > startIndex) + { + return part[startIndex..endIndex]; + } + } + } + } } - Log.Warning($"Unable to get GH runners for repo: [{response.StatusCode}] {response.ReasonPhrase}"); + return null; } + + public static async Task GetRunnerTokenForOrg(string githubToken, string orgName) { diff --git a/PoolManager.cs b/PoolManager.cs index c5a8657..28ccf40 100644 --- a/PoolManager.cs +++ b/PoolManager.cs @@ -119,14 +119,14 @@ private async Task ProcessStats(List targetConfig) GithubRunnersGauge.Labels(tgt.Name, "active").Set(0); GithubRunnersGauge.Labels(tgt.Name, "idle").Set(0); GithubRunnersGauge.Labels(tgt.Name, "offline").Set(0); - GitHubRunners orgRunners = tgt.Target switch + List orgRunners = tgt.Target switch { TargetType.Repository => await GitHubApi.GetRunnersForRepo(tgt.GitHubToken, tgt.Name), TargetType.Organization => await GitHubApi.GetRunnersForOrg(tgt.GitHubToken, tgt.Name), _ => throw new ArgumentOutOfRangeException() }; - var ghStatus = orgRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).GroupBy(x => + var ghStatus = orgRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).GroupBy(x => { if (x.Busy) { @@ -288,7 +288,7 @@ private async Task CleanUpRunners(List targetConfigs) _logger.LogInformation($"Cleaning runners for {githubTarget.Name}..."); // Get runner infos - GitHubRunners githubRunners = githubTarget.Target switch + List githubRunners = githubTarget.Target switch { TargetType.Organization => await GitHubApi.GetRunnersForOrg(githubTarget.GitHubToken, githubTarget.Name), TargetType.Repository => await GitHubApi.GetRunnersForRepo(githubTarget.GitHubToken, githubTarget.Name), @@ -296,7 +296,7 @@ private async Task CleanUpRunners(List targetConfigs) }; // Remove all offline runner entries from GitHub - List ghOfflineRunners = githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x.Status == "offline").ToList(); + List ghOfflineRunners = githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x.Status == "offline").ToList(); foreach (GitHubRunner runnerToRemove in ghOfflineRunners) { var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.Hostname == runnerToRemove.Name); @@ -337,7 +337,7 @@ private async Task CleanUpRunners(List targetConfigs) } // remove any long idling runners. pool manager will start fresh ones eventually if needed. Keeps em fresh. - List ghIdleRunners = githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x is { Status: "online", Busy: false }).ToList(); + List ghIdleRunners = githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x is { Status: "online", Busy: false }).ToList(); foreach (GitHubRunner ghIdleRunner in ghIdleRunners) { var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.Hostname == ghIdleRunner.Name); @@ -384,7 +384,7 @@ private async Task CleanUpRunners(List targetConfigs) TargetType.Repository => await GitHubApi.GetRunnersForRepo(githubTarget.GitHubToken, githubTarget.Name), _ => throw new ArgumentOutOfRangeException() }; - registeredServerNames.AddRange(githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).Select(x => x.Name)); + registeredServerNames.AddRange(githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).Select(x => x.Name)); } // Remove every VM that's not in the github registered runners