textvqa doc_to_target

EvolvingLMMs-Lab · Mar 31, 2024 · b202dad · b202dad
1 parent 0e26684
commit b202dad
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml b/lmms_eval/tasks/textvqa/_default_template_textvqa_yaml
@@ -2,7 +2,7 @@ dataset_path: lmms-lab/textvqa
 output_type: generate_until
 doc_to_visual: !function utils.textvqa_doc_to_visual
 doc_to_text: !function utils.textvqa_doc_to_text
-doc_to_target: "answer"
+doc_to_target: !function utils.textvqa_doc_to_target
 generation_kwargs:
   until:
     - "ASSISTANT:"

diff --git a/lmms_eval/tasks/textvqa/utils.py b/lmms_eval/tasks/textvqa/utils.py
@@ -1,5 +1,6 @@
 import re
 import os
+import random
 import json
 import yaml
 import pathlib
@@ -66,3 +67,7 @@ def textvqa_aggreate_submissions(results, args):
         json.dump(results, f)
     # print(f"Submission file saved to {path}")
     eval_logger.info(f"Submission file saved to {path}")
+
+def textvqa_doc_to_target(doc):
+    answers = doc["answers"]
+    return random.choice(answers) if isinstance(answers, list) else answers