fix steps check for anneal on first cycle (#1316)

axolotl-ai-cloud · Feb 27, 2024 · 2c9c88b · 2c9c88b
1 parent 5265cd6
commit 2c9c88b
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/axolotl/monkeypatch/relora.py b/src/axolotl/monkeypatch/relora.py
@@ -267,7 +267,7 @@ def get_lr(self) -> float:
         original = self.inner_schedule.get_lr()
         step = self.last_epoch
 
-        if step < self.relora_steps:
+        if step < self.relora_steps - self.warmup_steps:
             scale = 1
         else:
             per_relora_progress = step % self.relora_steps