From 38c03ad24dafc3c52ddb22de6451f996eec9a356 Mon Sep 17 00:00:00 2001
From: lewtun <lewis.c.tunstall@gmail.com>
Date: Wed, 14 Aug 2024 09:54:58 +0200
Subject: [PATCH] Fix IFEval metric (#259)

* Fix IFEval metric

* Remove dead code
---
 src/lighteval/tasks/extended/ifeval/main.py | 33 ++++++++++-----------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/lighteval/tasks/extended/ifeval/main.py b/src/lighteval/tasks/extended/ifeval/main.py
index 119a0403..69fe53c5 100644
--- a/src/lighteval/tasks/extended/ifeval/main.py
+++ b/src/lighteval/tasks/extended/ifeval/main.py
@@ -46,23 +46,6 @@ def ifeval_prompt(line, task_name: str = None):
     )
 
 
-# We create the task config
-ifeval = LightevalTaskConfig(
-    name="ifeval",
-    prompt_function=ifeval_prompt,
-    suite=["extended"],
-    hf_repo="wis-k/instruction-following-eval",
-    hf_subset="default",
-    metric=["ifeval_metric"],
-    hf_avail_splits=["train"],
-    evaluation_splits=["train"],
-    few_shots_split="train",
-    few_shots_select="random_sampling",
-    generation_size=1280,
-    stop_sequence=[],  # no stop sequence, will use eot token
-)
-
-
 submetric_names = [
     "prompt_level_strict_acc",
     "inst_level_strict_acc",
@@ -156,6 +139,22 @@ def agg_inst_level_acc(items):
     },
 )
 
+# We create the task config
+ifeval = LightevalTaskConfig(
+    name="ifeval",
+    prompt_function=ifeval_prompt,
+    suite=["extended"],
+    hf_repo="wis-k/instruction-following-eval",
+    hf_subset="default",
+    metric=[ifeval_metrics],
+    hf_avail_splits=["train"],
+    evaluation_splits=["train"],
+    few_shots_split="train",
+    few_shots_select="random_sampling",
+    generation_size=1280,
+    stop_sequence=[],  # no stop sequence, will use eot token
+)
+
 
 TASKS_TABLE = [ifeval]