Use only dataclasses for task init (#212)

* replaced json tasks by python tasks --------- Co-authored-by: Nathan Habib <[email protected]>
huggingface · Jul 9, 2024 · 3aaec22 · 3aaec22
1 parent 3f90950
commit 3aaec22
Show file tree

Hide file tree

Showing 13 changed files with 22,692 additions and 1,299 deletions.
diff --git a/community_tasks/_template.py b/community_tasks/_template.py
@@ -106,7 +106,7 @@ def prompt_fn(line, task_name: str = None):
 
 # STORE YOUR EVALS
 SUBSET_TASKS = [CustomSubsetTask(name=f"mytask:{subset}", hf_subset=subset) for subset in SAMPLE_SUBSETS]
-_TASKS = SUBSET_TASKS + [task]
+TASKS_TABLE = SUBSET_TASKS + [task]
 
 
 # CUSTOM METRIC IF NEEDED
@@ -124,8 +124,6 @@ def prompt_fn(line, task_name: str = None):
 # MODULE LOGIC
 # You should not need to touch this
 # Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in _TASKS]
-
 if __name__ == "__main__":
-    print(t["name"] for t in TASKS_TABLE)
+    print(t.name for t in TASKS_TABLE)
     print(len(TASKS_TABLE))
diff --git a/community_tasks/aimo_evals.py b/community_tasks/aimo_evals.py
@@ -55,14 +55,12 @@ def aimo_prompt(line, task_name: str = None):
 
 
 # STORE YOUR EVALS
-_TASKS = [task]
+TASKS_TABLE = [task]
 
 
 # MODULE LOGIC
 # You should not need to touch this
-# Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in _TASKS]
 
 if __name__ == "__main__":
-    print(t["name"] for t in TASKS_TABLE)
+    print(t.name for t in TASKS_TABLE)
     print(len(TASKS_TABLE))
diff --git a/community_tasks/arabic_evals.py b/community_tasks/arabic_evals.py
@@ -577,7 +577,7 @@ def sciq_prompt_arabic(line, task_name: str = None):
     )
 
 
-_TASKS = (
+TASKS_TABLE = (
     ARABIC_MMLU_TASKS
     + ACVA_TASKS
     + ALGHAFA_TASKS
@@ -595,9 +595,6 @@ def sciq_prompt_arabic(line, task_name: str = None):
     + [sciq_ar_task]
 )
 
-# Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in _TASKS]
-
 if __name__ == "__main__":
-    print(t["name"] for t in TASKS_TABLE)
+    print(t.name for t in TASKS_TABLE)
     print(len(TASKS_TABLE))
diff --git a/community_tasks/german_rag_evals.py b/community_tasks/german_rag_evals.py
@@ -219,14 +219,12 @@ def prompt_fn_context_question_match(line, task_name: str = None):
 
 
 # STORE YOUR EVALS
-_TASKS = [task1, task2, task3, task4]
+TASKS_TABLE = [task1, task2, task3, task4]
 
 
 # MODULE LOGIC
 # You should not need to touch this
-# Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in _TASKS]
 
 if __name__ == "__main__":
-    print(t["name"] for t in TASKS_TABLE)
+    print(t.name for t in TASKS_TABLE)
     print(len(TASKS_TABLE))
diff --git a/examples/nanotron/custom_evaluation_tasks.py b/examples/nanotron/custom_evaluation_tasks.py
@@ -679,7 +679,7 @@ def agi_eval_prompt_no_letters(line, task_name: str = None):
 EARLY_SIGNAL_TASKS = ",".join([t[1] for t in COMMON_SENSE_REASONING_STRING] + [t[1] for t in MMLU_STRING])
 
 # Convert to dict for lighteval
-TASKS_TABLE = [task.as_dict() for task in _TASKS]
+TASKS_TABLE = _TASKS
 # You can have a few pre-organised groups of tasks
 TASKS_GROUPS = {
     "all": ",".join(t[1] for t in _TASKS_STRINGS),