Merge pull request #3 from Bslabe123/backlog_metrics

Backlog metrics
Bslabe123 · Jun 24, 2024 · f91a0a3 · f91a0a3
2 parents 4637fd9 + 2845584
commit f91a0a3
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/jetstream/core/orchestrator.py b/jetstream/core/orchestrator.py
@@ -571,9 +571,11 @@ def _transfer_thread(self, idx: int):
       self._generate_backlogs[target_idx].put(new_request, block=True)
       logging.info(
           "Successfully transferred prefill "
-          "from prefill engine %d to generate engine %d.",
+          "from prefill engine %d to generate engine %d "
+          "(%d requests now in backlog).",
           idx,
           target_idx,
+          self._generate_backlogs[target_idx].qsize(),
       )
 
   def _generate_thread(self, idx: int):