add more API and some db job additions

ApiController.cs

@@ -13,6 +13,22 @@ public async Task<IResult> GetRunners()
         var recentRunners = await db.Runners.Include(x => x.Lifecycle).OrderByDescending(x => x.RunnerId).Take(100).ToListAsync();
         return Results.Json(recentRunners);
     }
+
+    [Route("get-jobs")]
+    public async Task<IResult> GetJobs()
+    {
+        var db = new ActionsRunnerContext();
+        var recentRunners = await db.Jobs.OrderByDescending(x => x.JobId).Take(100).ToListAsync();
+        return Results.Json(recentRunners);
+    }
+
+    [Route("get-runner/{runnerid}")]
+    public async Task<IResult> GetRunner(int runnerid)
+    {
+        var db = new ActionsRunnerContext();
+        var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.RunnerId == runnerid);
+        return Results.Json(runner);
+    }
 
     [Route("get-job/{jobid}")]
     public async Task<IResult> GetJob(int jobid)
@@ -22,6 +38,27 @@ public async Task<IResult> GetJob(int jobid)
         return Results.Json(job);
 
     }
+
+    [Route("get-potential-runners/{jobId}")]
+    public async Task<IResult> GetPotentialRunners(int jobId)
+    {
+        var db = new ActionsRunnerContext();
+        var job = await db.Jobs.FirstOrDefaultAsync(x => x.JobId == jobId);
+
+        // get labels
+        string size = job.RequestedSize;
+        string owner = job.Owner;
+        string profile = job.RequestedProfile;
+
+        var potentialRunners = db.Runners
+            .Include(x => x.Lifecycle)
+            .Where(x => x.Size == size && x.Owner == owner && x.Profile == profile)
+            .AsEnumerable()
+            .Where(x => x.LastState == RunnerStatus.Created || x.LastState == RunnerStatus.Provisioned)
+            .ToList();
+
+        return Results.Json(potentialRunners);
+    }
 
 
 }

Database/DbContext.cs

@@ -123,4 +123,6 @@ public class Job
     public int? RunnerId { get; set; }
     public Runner Runner { get; set; }
     public bool Orphan { get; set; }
+    public string RequestedProfile { get; set; }
+    public string RequestedSize { get; set; }
 }

Migrations/20240729091932_jobadditions.cs

@@ -0,0 +1,38 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace GithubActionsOrchestrator.Migrations
+{
+    /// <inheritdoc />
+    public partial class jobadditions : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "RequestedProfile",
+                table: "Jobs",
+                type: "text",
+                nullable: true);
+
+            migrationBuilder.AddColumn<string>(
+                name: "RequestedSize",
+                table: "Jobs",
+                type: "text",
+                nullable: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "RequestedProfile",
+                table: "Jobs");
+
+            migrationBuilder.DropColumn(
+                name: "RequestedSize",
+                table: "Jobs");
+        }
+    }
+}

Migrations/ActionsRunnerContextModelSnapshot.cs

@@ -54,6 +54,12 @@ protected override void BuildModel(ModelBuilder modelBuilder)
                     b.Property<string>("Repository")
                         .HasColumnType("text");
 
+                    b.Property<string>("RequestedProfile")
+                        .HasColumnType("text");
+
+                    b.Property<string>("RequestedSize")
+                        .HasColumnType("text");
+
                     b.Property<int?>("RunnerId")
                         .HasColumnType("integer");
 

PoolManager.cs

@@ -67,10 +67,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             if (DateTime.UtcNow - crudeTimer > TimeSpan.FromMinutes(cullMinutes))
             {
                 _logger.LogInformation("Cleaning runners...");
-                // update the world state for htz
-                allHtzSrvs = await _cc.GetAllServersFromCsp();
                 await CleanUpRunners(targetConfig);
                 await StartPoolRunners(targetConfig);
+                await CheckForStuckJobs(targetConfig);
                 crudeTimer = DateTime.UtcNow;
             }
 
@@ -221,6 +220,65 @@ private async Task StartPoolRunners(List<GithubTargetConfiguration> targetConfig
         }
     }
 
+    private async Task CheckForStuckJobs(List<GithubTargetConfiguration> targetConfig)
+    {
+        var db = new ActionsRunnerContext();
+        var stuckTime = DateTime.UtcNow - TimeSpan.FromMinutes(10);
+        var stuckJobs = db.Jobs.Where(x => x.RunnerId == null && x.QueueTime < stuckTime).AsEnumerable();
+        foreach (var stuckJob in stuckJobs)
+        {
+            _logger.LogWarning($"Found stuck Job: {stuckJob.JobId} in {stuckJob.Repository}. Starting new runner to compensate...");
+
+            var owner = targetConfig.FirstOrDefault(x => x.Name == stuckJob.Owner);
+            if (owner == null)
+            {
+                _logger.LogError($"Unable to get owner for stuck job. {stuckJob.JobId}");
+                continue;
+            }
+
+            string runnerToken = owner.Target switch
+            {
+                TargetType.Repository => await GitHubApi.GetRunnerTokenForRepo(owner.GitHubToken, owner.Name),
+                TargetType.Organization => await GitHubApi.GetRunnerTokenForOrg(owner.GitHubToken, owner.Name),
+                _ => throw new ArgumentOutOfRangeException()
+            };
+            var profile = stuckJob.RequestedProfile ?? "default";
+            string arch = Program.Config.Sizes.FirstOrDefault(x => x.Name == stuckJob.RequestedSize)?.Arch;
+            Runner newRunner = new()
+            {
+                Size = stuckJob.RequestedSize,
+                Cloud = "htz",
+                Hostname = "Unknown",
+                Profile = profile,
+                Lifecycle =
+                [
+                    new RunnerLifecycle
+                    {
+                        EventTimeUtc = DateTime.UtcNow,
+                        Status = RunnerStatus.CreationQueued,
+                        Event = $"Created for stuck job {stuckJob.JobId}"
+                    }
+                ],
+                IsOnline = false,
+                Arch = arch,
+                IPv4 = string.Empty,
+                IsCustom = profile != "default",
+                Owner = stuckJob.Owner
+            };
+            await db.Runners.AddAsync(newRunner);
+            await db.SaveChangesAsync();
+
+            _queues.CreateTasks.Enqueue(new CreateRunnerTask
+            {
+                RunnerToken = runnerToken,
+                RepoName = stuckJob.Repository,
+                TargetType = owner.Target,
+                RunnerDbId = newRunner.RunnerId,
+
+            }); 
+        }
+    }
+
     private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
     {
         List<string> registeredServerNames = new();
@@ -340,8 +398,18 @@ private async Task CleanUpRunners(List<GithubTargetConfiguration> targetConfigs)
             }
 
             var runner = await db.Runners.Include(x => x.Lifecycle).FirstOrDefaultAsync(x => x.CloudServerId == htzSrv.Id);
-
-            if (runner.LastState >= RunnerStatus.Provisioned && DateTime.UtcNow - runner.LastStateTime > TimeSpan.FromMinutes(5))
+            if (runner.Lifecycle.Any(x => x.Status == RunnerStatus.DeletionQueued))
+            {
+                runner.Lifecycle.Add(new()
+                {
+                    Status = RunnerStatus.DeletionQueued,
+                    Event = "Don't queue deletion due to Github registration. Runner already queued for deletion.",
+                    EventTimeUtc = DateTime.UtcNow
+                });
+                await db.SaveChangesAsync();
+
+            }
+            else if (runner.LastState >= RunnerStatus.Provisioned && DateTime.UtcNow - runner.LastStateTime > TimeSpan.FromMinutes(5))
             {
                 _logger.LogInformation($"Removing VM that is not in any GitHub registration: {htzSrv.Name} created at {htzSrv.Created:u}");
                 runner.IsOnline = false;