Still log something on the other ranks

allenai · Mar 11, 2024 · 696afb5 · 696afb5
1 parent c6ad0c1
commit 696afb5
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/olmo/train.py b/olmo/train.py
@@ -958,8 +958,11 @@ def on_trace_ready(p):
                         metrics.update(lr_monitor.check())
 
                     # Log metrics to console.
-                    if self.global_step % self.cfg.console_log_interval == 0 and get_global_rank() == 0:
-                        self.log_metrics_to_console(f"[step={self.global_step}/{self.max_steps}]", metrics)
+                    if self.global_step % self.cfg.console_log_interval == 0:
+                        if get_global_rank() == 0:
+                            self.log_metrics_to_console(f"[step={self.global_step}/{self.max_steps}]", metrics)
+                        else:
+                            log.info(f"[step={self.global_step}/{self.max_steps}]")
 
                     # Log metrics to W&B.
                     if (