[dashboard] patch tests that relies on timing (#45592)

cherrypick #45561 test-only patch; does not change the code under release. Signed-off-by: Lonnie Liu <[email protected]>
ray-project · May 28, 2024 · 68ffbe5 · 68ffbe5
1 parent ec16946
commit 68ffbe5
Showing 1 changed file with 9 additions and 5 deletions.
diff --git a/dashboard/tests/test_dashboard.py b/dashboard/tests/test_dashboard.py
@@ -1318,12 +1318,16 @@ async def make_blocking_call():
     addr = ray_context["raylet_ip_address"]
     prom_addresses = [f"{addr}:{dashboard_consts.DASHBOARD_METRIC_PORT}"]
 
-    metrics_samples: Dict[str, List[Sample]] = fetch_prometheus_metrics(prom_addresses)
-    print(metrics_samples)
+    def check_lag_metrics():
+        metrics_samples: Dict[str, List[Sample]] = fetch_prometheus_metrics(
+            prom_addresses
+        )
+        lag_metric_samples = metrics_samples["ray_dashboard_event_loop_lag_seconds"]
+        assert len(lag_metric_samples) > 0
+        assert any(sample.value > 1 for sample in lag_metric_samples)
+        return True
 
-    lag_metric_samples = metrics_samples["ray_dashboard_event_loop_lag_seconds"]
-    assert len(lag_metric_samples) > 0
-    assert any(sample.value > 1 for sample in lag_metric_samples)
+    wait_for_condition(check_lag_metrics)
 
 
 if __name__ == "__main__":