From 87a8b7020b511245690fd28f9ccc4db7465aefc2 Mon Sep 17 00:00:00 2001
From: epwalsh <petew@allenai.org>
Date: Mon, 11 Dec 2023 13:41:59 -0800
Subject: [PATCH] fix-tune down to 0 LR

---
 configs/mcli/v1_5-mix-medium-mitch-ish.yaml | 23 +++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/configs/mcli/v1_5-mix-medium-mitch-ish.yaml b/configs/mcli/v1_5-mix-medium-mitch-ish.yaml
index 9aa979078..9b67cdc2e 100644
--- a/configs/mcli/v1_5-mix-medium-mitch-ish.yaml
+++ b/configs/mcli/v1_5-mix-medium-mitch-ish.yaml
@@ -6,7 +6,8 @@ gpu_type: a100_40gb
 integrations:
   - integration_type: git_repo
     git_repo: allenai/LLM
-    git_branch: mitchish
+    # git_branch: mitchish
+    git_commit: 148ca062e7f1f7667d7fc0f4346e97467e66ce87
     pip_install: -e .
     ssh_clone: true
 command: |-
@@ -28,11 +29,21 @@ command: |-
   --nproc_per_node 8 \
   scripts/train.py configs/v1_5-mix-medium-mitch-ish-s3.yaml \
     --run_name=v1_5-mix-mitch-ish \
-    --wandb.name=v1_5-mix-mitch-ish-mcli \
+    --wandb.name=v1_5-mix-mitch-ish-mcli-final \
     --global_train_batch_size=2160 \
     --time_limit=169200 \
-    --load_path='${path.last_checkpoint:s3://ai2-llm/checkpoints/7b/v1_5-mix-mitch-ish}' \
+    --eval_interval=100 \
+    --save_interval=500 \
+    --load_path=s3://ai2-llm/checkpoints/7b/v1_5-mix-mitch-ish/step556000 \
+    --remote_save_folder=s3://ai2-llm/checkpoints/7b/v1_5-mix-mitch-ish-final \
     --epoch=1 \
-    --optimizer.learning_rate=0.00003 \
-    --scheduler.t_warmup=501000 \
-    --scheduler.t_max=700000
+    --optimizer.learning_rate=0.000023 \
+    --scheduler.t_warmup=556000 \
+    --scheduler.t_max=557000 \
+    --scheduler.alpha_f=0.001 \
+    --stop_at=557001
+
+# --load_path='${path.last_checkpoint:s3://ai2-llm/checkpoints/7b/v1_5-mix-mitch-ish}' \
+# --optimizer.learning_rate=0.00003 \
+# --scheduler.t_warmup=501000 \
+# --scheduler.t_max=700000