Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add Muirbench (#143)
Browse files Browse the repository at this point in the history
* handle gen kwargs in internvl2

* Add muirbench
kcz358 committed Sep 5, 2024
1 parent de82ca0 commit 1f73767
Showing 3 changed files with 22 additions and 18 deletions.
2 changes: 1 addition & 1 deletion lmms_eval/models/internvl2.py
Original file line number Diff line number Diff line change
@@ -248,7 +248,7 @@ def generate_until(self, requests) -> List[str]:
for k, v in gen_kwargs.items():
if k not in DEFAULT_GEN_KWARGS:
pop_keys.append(k)

for k in pop_keys:
gen_kwargs.pop(k)

3 changes: 2 additions & 1 deletion lmms_eval/tasks/muirbench/muirbench.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@

dataset_path: MUIRBENCH/MUIRBENCH
task: "muirbench"
dataset_kwargs:
@@ -9,7 +10,7 @@ doc_to_text: !function utils.muir_doc_to_text
doc_to_target: !function utils.muir_doc_to_target
process_results: !function utils.muir_process_results

lmms_eval_specific_kwargs:
model_specific_prompt_kwargs:
default:
pre_prompt: ""
post_prompt: "\nAnswer with the option's letter from the given choices directly."
35 changes: 19 additions & 16 deletions lmms_eval/tasks/muirbench/utils.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,23 @@
import re

import pandas as pd

from lmms_eval.filters.extraction import ExtendedRegexFilter
from lmms_eval.filters.transformation import MapFilter
import re
import pandas as pd


def muir_doc_to_text(doc, lmms_eval_specific_kwargs=None):
def muir_doc_to_text(doc, model_specific_prompt_kwargs=None):
question, choices = doc["question"], doc["options"]
len_choices = len(choices)
post_prompt = lmms_eval_specific_kwargs["post_prompt"]
pre_prompt = lmms_eval_specific_kwargs["pre_prompt"]
post_prompt = model_specific_prompt_kwargs["post_prompt"]
pre_prompt = model_specific_prompt_kwargs["pre_prompt"]
options = [chr(ord("A") + i) for i in range(len_choices)]
choices_str = "\n".join([f"{option}. {choice}" for option, choice in zip(options, choices)])
return f"{pre_prompt}{question}\n{choices_str}{post_prompt}"


def muir_doc_to_visual(doc):
image_list = [image.convert("RGB") for image in doc["image_list"]]
return image_list
return image_list


def muir_doc_to_target(doc):
@@ -34,15 +33,15 @@ def muir_process_results(doc, result):
image_type = doc["image_type"]

data_dict = {
"pred": pred,
"task": task,
"idx": idx,
"image_relation": image_relation,
"answer": answer,
"image_type": image_type,
}
"pred" : pred,
"task" : task,
"idx" : idx,
"image_relation" : image_relation,
"answer" : answer,
"image_type" : image_type,
}

return {"muirbench_score_overall": data_dict}
return {"muirbench_score_overall" : data_dict}


def muir_aggregation(results):
@@ -62,15 +61,19 @@ def muir_aggregation(results):
task_num[result["task"]] += 1

score = score / len(results)
task_score = {k: v / task_num[k] for k, v in task_score.items()}

task_score = {k : v / task_num[k] for k,v in task_score.items()}

print("=" * 50)
for k, v in task_score.items():
print(f"{k} : {v:.2f}")
print("=" * 50)

return score




class MultiChoiceRegexFilter(ExtendedRegexFilter):
def __init__(self, *args, **kwargs):
"""

0 comments on commit 1f73767

Please sign in to comment.