From 7c8ef0731b5237dbfc9de5d99f7db789b33ef8c3 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Tue, 27 Feb 2024 13:53:07 +0000 Subject: [PATCH] wip --- .../custom_tasks_with_custom_metrics/ifeval/ifeval.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tasks_examples/custom_tasks_with_custom_metrics/ifeval/ifeval.py b/tasks_examples/custom_tasks_with_custom_metrics/ifeval/ifeval.py index dc39694cc..3b4452d63 100644 --- a/tasks_examples/custom_tasks_with_custom_metrics/ifeval/ifeval.py +++ b/tasks_examples/custom_tasks_with_custom_metrics/ifeval/ifeval.py @@ -25,18 +25,17 @@ few_shots_split="train", few_shots_select="random_sampling", generation_size=1280, - stop_sequence=[], # no stop sequence + stop_sequence=[], # no stop sequence, will use eot token ) +# very specific task where there are no precise outputs but instead we test if the format obeys rules def ifeval_prompt(line, task_name: str = None): return Doc( task_name=task_name, query=line["prompt"], - choices=[ - None - ], # very specific task where there are no precise outputs but instead we test if the format obeys rules - gold_index=0, # very specific task where there are no precise outputs but instead we test if the format obeys rules + choices=[""], + gold_index=0, instruction="", specific={"instructions_id_list": line["instruction_id_list"], "kwargs": line["kwargs"]}, )