From c0a638ad71a63cfb22d6febe044359fe3c758e62 Mon Sep 17 00:00:00 2001
From: epwalsh <petew@allenai.org>
Date: Wed, 27 Mar 2024 17:02:21 -0700
Subject: [PATCH] add more MMLU variants to config

---
 configs/mcli/mitchish70.yaml |  3 ++-
 configs/mitchish70-s3.yaml   | 12 ++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/configs/mcli/mitchish70.yaml b/configs/mcli/mitchish70.yaml
index ac1bb431e..b6b725985 100644
--- a/configs/mcli/mitchish70.yaml
+++ b/configs/mcli/mitchish70.yaml
@@ -52,7 +52,7 @@ command: |-
   scripts/train.py configs/mitchish70-s3.yaml \
     --run_name=mitchish70-002 \
     --wandb.group=mitchish70-official \
-    --load_path=s3://ai2-llm/checkpoints/OLMo-large/mitchish70-002/step34700-unsharded \
+    '--load_path=${path.last_checkpoint:${remote_save_folder}}' \
     --global_train_batch_size=1536 \
     --device_train_microbatch_size=3 \
     --time_limit=604800 \
@@ -64,3 +64,4 @@ command: |-
 #    --load_path=s3://ai2-llm/checkpoints/OLMo-large/mitchish70-002/step32050-unsharded \
 #    --load_path=s3://ai2-llm/checkpoints/OLMo-large/mitchish70-002/step32300-unsharded \
 #    --load_path=s3://ai2-llm/checkpoints/OLMo-large/mitchish70-002/step32300 \
+#    --load_path=s3://ai2-llm/checkpoints/OLMo-large/mitchish70-002/step34700-unsharded \
diff --git a/configs/mitchish70-s3.yaml b/configs/mitchish70-s3.yaml
index 1bc5e579e..15fc89d07 100644
--- a/configs/mitchish70-s3.yaml
+++ b/configs/mitchish70-s3.yaml
@@ -189,6 +189,18 @@ evaluators:
   - label: mmlu_other_var
     type: downstream
 
+  - label: mmlu_stem_mc_5shot
+    type: downstream
+
+  - label: mmlu_humanities_mc_5shot
+    type: downstream
+
+  - label: mmlu_social_sciences_mc_5shot
+    type: downstream
+
+  - label: mmlu_other_mc_5shot
+    type: downstream
+
 data:
   pad_direction: right
   num_workers: 8