Skip to content

Commit

Permalink
add pagination support for githubapi
Browse files Browse the repository at this point in the history
  • Loading branch information
@elasticroentgen
elasticroentgen committed Jul 29, 2024
1 parent 695b83f commit b738927
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 33 deletions.
80 changes: 53 additions & 27 deletions GitHubApi.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,17 @@
namespace GithubActionsOrchestrator;

public static class GitHubApi {
public static async Task<GitHubRunners> GetRunnersForOrg(string githubToken, string orgName)
public static async Task<List<GitHubRunner>> GetRunnersForOrg(string githubToken, string orgName)
{
return await GetRunners(githubToken, $"orgs/{orgName}");
}

public static async Task<List<GitHubRunner>> GetRunnersForRepo(string githubToken, string repoName)
{
return await GetRunners(githubToken, $"repos/{repoName}");
}

private static async Task<List<GitHubRunner>> GetRunners(string githubToken, string ownerPath)
{

// Register a runner with github
Expand All @@ -18,42 +28,58 @@ public static async Task<GitHubRunners> GetRunnersForOrg(string githubToken, str
client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"Bearer {githubToken}");
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("hetzner-autoscale", "1"));

HttpResponseMessage response = await client.GetAsync(
$"https://api.github.com/orgs/{orgName}/actions/runners");

if(response.IsSuccessStatusCode)
var runners = new List<GitHubRunner>();
string url = $"https://api.github.com/{ownerPath}/actions/runners?per_page=100";

while (!string.IsNullOrEmpty(url))
{
string content = await response.Content.ReadAsStringAsync();
GitHubRunners responseObject = JsonSerializer.Deserialize<GitHubRunners>(content);
return responseObject;
HttpResponseMessage response = await client.GetAsync(url);
if (response.IsSuccessStatusCode)
{
string content = await response.Content.ReadAsStringAsync();
var pageRunners = JsonSerializer.Deserialize<GitHubRunners>(content);
if (pageRunners != null)
{
runners.AddRange(pageRunners.Runners);
}

url = GetNextPageUrl(response);
}
else
{
Log.Warning($"Unable to get GH runners for org: [{response.StatusCode}] {response.ReasonPhrase}");
}
}

Log.Warning($"Unable to get GH runners for org: [{response.StatusCode}] {response.ReasonPhrase}");

return null;
return runners;
}
public static async Task<GitHubRunners> GetRunnersForRepo(string githubToken, string repoName)
private static string GetNextPageUrl(HttpResponseMessage response)
{

// Register a runner with github
using HttpClient client = new();
client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/vnd.github+json"));
client.DefaultRequestHeaders.Add("X-GitHub-Api-Version", "2022-11-28");
client.DefaultRequestHeaders.Authorization = AuthenticationHeaderValue.Parse($"Bearer {githubToken}");
client.DefaultRequestHeaders.UserAgent.Add(new ProductInfoHeaderValue("hetzner-autoscale", "1"));

HttpResponseMessage response = await client.GetAsync(
$"https://api.github.com/repos/{repoName}/actions/runners");

if(response.IsSuccessStatusCode)
if (response.Headers.TryGetValues("Link", out var links))
{
string content = await response.Content.ReadAsStringAsync();
GitHubRunners responseObject = JsonSerializer.Deserialize<GitHubRunners>(content);
return responseObject;
foreach (var link in links)
{
// Example Link header format: <https://api.github.com/resource?page=2>; rel="next", ...
foreach (var part in link.Split(','))
{
if (part.Contains("rel=\"next\""))
{
int startIndex = part.IndexOf('<') + 1;
int endIndex = part.IndexOf('>');
if (startIndex >= 0 && endIndex > startIndex)
{
return part[startIndex..endIndex];
}
}
}
}
}
Log.Warning($"Unable to get GH runners for repo: [{response.StatusCode}] {response.ReasonPhrase}");

return null;
}


public static async Task<string> GetRunnerTokenForOrg(string githubToken, string orgName)
{

Expand Down
12 changes: 6 additions & 6 deletions PoolManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,14 +119,14 @@ private async Task ProcessStats(List<GithubTargetConfiguration> targetConfig)
GithubRunnersGauge.Labels(tgt.Name, "active").Set(0);
GithubRunnersGauge.Labels(tgt.Name, "idle").Set(0);
GithubRunnersGauge.Labels(tgt.Name, "offline").Set(0);
GitHubRunners orgRunners = tgt.Target switch
List<GitHubRunner> orgRunners = tgt.Target switch
{
TargetType.Repository => await GitHubApi.GetRunnersForRepo(tgt.GitHubToken, tgt.Name),
TargetType.Organization => await GitHubApi.GetRunnersForOrg(tgt.GitHubToken, tgt.Name),
_ => throw new ArgumentOutOfRangeException()
};

var ghStatus = orgRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).GroupBy(x =>
var ghStatus = orgRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).GroupBy(x =>
{
if (x.Busy)
{
Expand Down Expand Up @@ -288,15 +288,15 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
_logger.LogInformation($"Cleaning runners for {githubTarget.Name}...");

// Get runner infos
GitHubRunners githubRunners = githubTarget.Target switch
List<GitHubRunner> githubRunners = githubTarget.Target switch
{
TargetType.Organization => await GitHubApi.GetRunnersForOrg(githubTarget.GitHubToken, githubTarget.Name),
TargetType.Repository => await GitHubApi.GetRunnersForRepo(githubTarget.GitHubToken, githubTarget.Name),
_ => throw new ArgumentOutOfRangeException()
};

// Remove all offline runner entries from GitHub
List<GitHubRunner> ghOfflineRunners = githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x.Status == "offline").ToList();
List<GitHubRunner> ghOfflineRunners = githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x.Status == "offline").ToList();
foreach (GitHubRunner runnerToRemove in ghOfflineRunners)
{
var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.Hostname == runnerToRemove.Name);
Expand Down Expand Up @@ -337,7 +337,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
}

// remove any long idling runners. pool manager will start fresh ones eventually if needed. Keeps em fresh.
List<GitHubRunner> ghIdleRunners = githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x is { Status: "online", Busy: false }).ToList();
List<GitHubRunner> ghIdleRunners = githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix) && x is { Status: "online", Busy: false }).ToList();
foreach (GitHubRunner ghIdleRunner in ghIdleRunners)
{
var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.Hostname == ghIdleRunner.Name);
Expand Down Expand Up @@ -384,7 +384,7 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
TargetType.Repository => await GitHubApi.GetRunnersForRepo(githubTarget.GitHubToken, githubTarget.Name),
_ => throw new ArgumentOutOfRangeException()
};
registeredServerNames.AddRange(githubRunners.Runners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).Select(x => x.Name));
registeredServerNames.AddRange(githubRunners.Where(x => x.Name.StartsWith(Program.Config.RunnerPrefix)).Select(x => x.Name));
}

// Remove every VM that's not in the github registered runners
Expand Down

0 comments on commit b738927

Please sign in to comment.