add arc_challenge_mt

EleutherAI · May 29, 2024 · de3eff9 · de3eff9
1 parent b4cd85d
commit de3eff9
Show file tree

Hide file tree

Showing 11 changed files with 53 additions and 0 deletions.
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_da.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_da.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_da
+dataset_name: da
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_de.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_de.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_de
+dataset_name: de
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_el.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_el.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_el
+dataset_name: el
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_es.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_es.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_es
+dataset_name: es
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml
@@ -0,0 +1,23 @@
+group:
+  - arc_challenge_mt
+task: arc_challenge_mt_fi
+dataset_path: LumiOpen/arc_challenge_mt
+dataset_name: fi
+output_type: multiple_choice
+training_split: train
+validation_split: validation
+test_split: test
+doc_to_text: "Question: {{question}}\nAnswer:"
+doc_to_target: "{{choices.label.index(answerKey)}}"
+doc_to_choice: "{{choices.text}}"
+should_decontaminate: true
+doc_to_decontamination_query: "Question: {{question}}\nAnswer:"
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: acc_norm
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_hu.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_hu.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_hu
+dataset_name: hu
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_it.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_it.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_it
+dataset_name: it
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_nb.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_nb.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_nb
+dataset_name: nb
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_pl.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_pl.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_pl
+dataset_name: pl
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_pt.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_pt.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_pt
+dataset_name: pt
diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_sv.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_sv.yaml
@@ -0,0 +1,3 @@
+include: arc_challenge_mt_fi.yaml
+task: arc_challenge_mt_sv
+dataset_name: sv