limit retries

EthDevOps · May 14, 2024 · dab13e2 · dab13e2
1 parent 54cf27e
commit dab13e2
Showing 1 changed file with 17 additions and 2 deletions.
diff --git a/PoolManager.cs b/PoolManager.cs
@@ -354,7 +354,15 @@ private async Task<bool> DeleteRunner(DeleteRunnerTask rt)
             _logger.LogError(
                 $"Unable to delete runner [{rt.ServerId} | Retry: {rt.RetryCount}]: {ex.Message}");
             rt.RetryCount += 1;
-            _queues.DeleteTasks.Enqueue(rt);
+            if (rt.RetryCount < 10)
+            {
+                _queues.DeleteTasks.Enqueue(rt);
+            }
+            else
+            {
+                _logger.LogError($"Retries exceeded for {rt.ServerId}. Giving up.");
+            }
+
             return false;
         }
     }
@@ -378,7 +386,14 @@ private async Task<bool> CreateRunner(CreateRunnerTask rt)
         {
             _logger.LogError($"Unable to create runner [{rt.Size} on {rt.Arch} | Retry: {rt.RetryCount}]: {ex.Message}");
             rt.RetryCount += 1;
-            _queues.CreateTasks.Enqueue(rt);
+            if (rt.RetryCount < 10)
+            {
+                _queues.CreateTasks.Enqueue(rt);
+            }
+            else
+            {
+                _logger.LogError($"Retries exceeded for {rt.Size} on {rt.Arch}. giving up.");
+            }
             return false;
         }
     }