diff --git a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml index fd485002..5c6205e9 100644 --- a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml +++ b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml @@ -2,7 +2,7 @@ dataset_path: lmms-lab/textvqa output_type: generate_until doc_to_visual: !function utils.textvqa_doc_to_visual doc_to_text: !function utils.textvqa_doc_to_text -doc_to_target: "answer" +doc_to_target: !function utils.textvqa_doc_to_target generation_kwargs: until: - "ASSISTANT:" diff --git a/lmms_eval/tasks/textvqa/utils.py b/lmms_eval/tasks/textvqa/utils.py index ea3b503b..dd87625d 100644 --- a/lmms_eval/tasks/textvqa/utils.py +++ b/lmms_eval/tasks/textvqa/utils.py @@ -1,5 +1,6 @@ import re import os +import random import json import yaml import pathlib @@ -66,3 +67,7 @@ def textvqa_aggreate_submissions(results, args): json.dump(results, f) # print(f"Submission file saved to {path}") eval_logger.info(f"Submission file saved to {path}") + +def textvqa_doc_to_target(doc): + answers = doc["answers"] + return random.choice(answers) if isinstance(answers, list) else answers