add runner backchannel for provisioning

EthDevOps · May 7, 2024 · 1b1c57a · 1b1c57a
1 parent 3d796de
commit 1b1c57a
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 3 deletions.
diff --git a/CloudController.cs b/CloudController.cs
@@ -138,7 +138,9 @@ public async Task<string> CreateNewRunner(string arch, string size, string runne
             CreatedAt = DateTime.UtcNow,
             TargetName = targetName,
             Size = size,
-            Arch = arch
+            Arch = arch,
+            Profile = profileName,
+            IsCustom = isCustom
         });
         StoreActiveRunners();
         return newSrv.Name;
@@ -278,4 +280,9 @@ public List<Machine> GetRunnersForTarget(string orgName)
     {
         return _activeRunners.Where(x => x.TargetName == orgName).ToList();
     }
+
+    public Machine GetRunnerByHostname(string hostname)
+    {
+        return _activeRunners.FirstOrDefault(x => x.Name == hostname);
+    }
 }
diff --git a/Machine.cs b/Machine.cs
@@ -13,4 +13,6 @@ public class Machine
     public string TargetName { get; set; }
     public string Size { get; set; }
     public string Arch { get; set; }
+    public string Profile { get; set; }
+    public bool IsCustom { get; set; }
 }
diff --git a/Program.cs b/Program.cs
@@ -108,9 +108,51 @@ public static void Main(string[] args)
 
         WebApplication app = builder.Build();
 
+        app.MapPost("/runner-state", async (HttpRequest request, 
+            [FromServices] CloudController cloud, [FromServices] ILogger<Program> logger, [FromServices] RunnerQueue runnerQueue, [FromQuery] string hostname, [FromQuery] string state) =>
+        {
+            switch (state)
+            {
+                case "ok":
+                    // Remove from provisioning dict
+                    Log.Information($"Runner {hostname} finished provisioning.");
+                    runnerQueue.CreatedRunners.Remove(hostname, out _);
+                    break;
+                case "error":
+                    Log.Warning($"Runner {hostname} failed provisioning.");
+                    if (request.Form.Files.Count > 0)
+                    {
+                        // Read the log file into a string
+                        using var reader = new StreamReader(request.Form.Files[0].OpenReadStream());
+                        string fileContent = await reader.ReadToEndAsync();
+                        Log.Information($"LOGS FROM {hostname}\n\n{fileContent}");    
+                    }
+                    
+                    // Get runner specs
+                    Machine runner = cloud.GetRunnerByHostname(hostname);
+                    
+                    // Queue creation of a new runner
+                    if (!runnerQueue.CreatedRunners.Remove(hostname, out CreateRunnerTask runnerSpec))
+                    {
+                       logger.LogError($"Unable to get previous settings for {hostname}");
+                    }
+                    
+                    logger.LogInformation($"Re-creating runner of type [{runnerSpec.Size}, {runnerSpec.Arch}] for {runnerSpec.RepoName}");
+                    runnerQueue.CreateTasks.Enqueue(runnerSpec);
+                    
+                    // Queue deletion of the failed runner
+                    runnerQueue.DeleteTasks.Enqueue(new DeleteRunnerTask
+                    {
+                        ServerId = runner.Id 
+                    });
+                    break;
+            }
+
+            return Results.StatusCode(201);
+        });
+
         // Prepare pools
-        app.MapPost("/github-webhook", async (HttpRequest request, [FromServices] CloudController cloud,
-            [FromServices] ILogger<Program> logger, [FromServices] RunnerQueue poolMgr) =>
+        app.MapPost("/github-webhook", async (HttpRequest request, [FromServices] CloudController cloud, [FromServices] ILogger<Program> logger, [FromServices] RunnerQueue poolMgr) =>
         {
             // Verify webhook HMAC - TODO
 
@@ -317,6 +359,8 @@ private static async Task JobQueued(ILogger<Program> logger, string repoName, Li
             return;
         }
 
+
+
         poolMgr.CreateTasks.Enqueue(new CreateRunnerTask
         {
             Arch = arch,

diff --git a/RunnerQueue.cs b/RunnerQueue.cs
@@ -6,4 +6,6 @@ public class RunnerQueue
 {
     public ConcurrentQueue<CreateRunnerTask> CreateTasks { get; } = new();
     public ConcurrentQueue<DeleteRunnerTask> DeleteTasks { get; } = new();
+
+    public ConcurrentDictionary<string, CreateRunnerTask> CreatedRunners { get; } = new();
 }