diff --git a/README.md b/README.md index e6a08bda..28edadb4 100755 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # LIME: LESS IS MORE FOR MLLM EVALUATION ## Annoucement -- [2024-10] 📰 We have released both the [LIME](https://huggingface.co/LIME-ICLR) dataset and the data duration pipeline! +- [2024-10] 📰 We have released both the [LIME](https://huggingface.co/LIME-DATA) dataset and the data duration pipeline! - [2024-09] 🍋 We have open-sourced the evaluation data and corresponding evaluation code for `LIME`. The data duration pipeline for LIME will be open-sourced within two weeks. ## Introduction @@ -22,7 +22,7 @@ cd lmms-eval pip install -e . ``` ### 2. download dataset from huggingface -download all datasets from [here](https://huggingface.co/LIME-ICLR) +download all datasets from [here](https://huggingface.co/LIME-DATA) ### 3.run evaluation #### For MLLMs evaluation: diff --git a/lmms_eval/tasks/ai2d_suit/ai2d_suit.yaml b/lmms_eval/tasks/ai2d_suit/ai2d_suit.yaml index 54821b95..3c9183f3 100755 --- a/lmms_eval/tasks/ai2d_suit/ai2d_suit.yaml +++ b/lmms_eval/tasks/ai2d_suit/ai2d_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ai2d +dataset_path: LIME-DATA/ai2d task: "ai2d_suit" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/ai2d_suit_VD/ai2d_suit.yaml b/lmms_eval/tasks/ai2d_suit_VD/ai2d_suit.yaml index 3c5423c8..a9af7098 100755 --- a/lmms_eval/tasks/ai2d_suit_VD/ai2d_suit.yaml +++ b/lmms_eval/tasks/ai2d_suit_VD/ai2d_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ai2d +dataset_path: LIME-DATA/ai2d task: "ai2d_suit_vd" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/chartqa_suit/chartqa.yaml b/lmms_eval/tasks/chartqa_suit/chartqa.yaml index 50bcba29..789f8b35 100755 --- a/lmms_eval/tasks/chartqa_suit/chartqa.yaml +++ b/lmms_eval/tasks/chartqa_suit/chartqa.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ChartQA +dataset_path: LIME-DATA/ChartQA dataset_kwargs: token: True task: "chartqa_suit" diff --git a/lmms_eval/tasks/chartqa_suit_vd/chartqa.yaml b/lmms_eval/tasks/chartqa_suit_vd/chartqa.yaml index 706f4fa1..3b8d39f3 100755 --- a/lmms_eval/tasks/chartqa_suit_vd/chartqa.yaml +++ b/lmms_eval/tasks/chartqa_suit_vd/chartqa.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ChartQA +dataset_path: LIME-DATA/ChartQA dataset_kwargs: token: True task: "chartqa_suit_vd" diff --git a/lmms_eval/tasks/coco_cap_suit/coco2017_cap_val.yaml b/lmms_eval/tasks/coco_cap_suit/coco2017_cap_val.yaml index 4860aa42..0ab16e3d 100755 --- a/lmms_eval/tasks/coco_cap_suit/coco2017_cap_val.yaml +++ b/lmms_eval/tasks/coco_cap_suit/coco2017_cap_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/COCO-Caption2017 +dataset_path: LIME-DATA/COCO-Caption2017 dataset_kwargs: token: True task: "coco2017_cap_suit_val" diff --git a/lmms_eval/tasks/coco_cap_suit_vd/coco2017_cap_val.yaml b/lmms_eval/tasks/coco_cap_suit_vd/coco2017_cap_val.yaml index 19f15624..ddc49749 100755 --- a/lmms_eval/tasks/coco_cap_suit_vd/coco2017_cap_val.yaml +++ b/lmms_eval/tasks/coco_cap_suit_vd/coco2017_cap_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/COCO-Caption2017 +dataset_path: LIME-DATA/COCO-Caption2017 dataset_kwargs: token: True task: "coco2017_cap_suit_val_vd" diff --git a/lmms_eval/tasks/infovqa_suit/_default_template_infovqa_yaml b/lmms_eval/tasks/infovqa_suit/_default_template_infovqa_yaml index cf55385a..92461783 100755 --- a/lmms_eval/tasks/infovqa_suit/_default_template_infovqa_yaml +++ b/lmms_eval/tasks/infovqa_suit/_default_template_infovqa_yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/infovqa +dataset_path: LIME-DATA/infovqa task: "infovqa_suit_val" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/infovqa_suit/infovqa_suit_val.yaml b/lmms_eval/tasks/infovqa_suit/infovqa_suit_val.yaml index b6878640..73b2e229 100755 --- a/lmms_eval/tasks/infovqa_suit/infovqa_suit_val.yaml +++ b/lmms_eval/tasks/infovqa_suit/infovqa_suit_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/infovqa +dataset_path: LIME-DATA/infovqa task: "infovqa_suit_val" test_split: train output_type: generate_until diff --git a/lmms_eval/tasks/infovqa_suit_VD/_default_template_infovqa_yaml b/lmms_eval/tasks/infovqa_suit_VD/_default_template_infovqa_yaml index cf55385a..92461783 100755 --- a/lmms_eval/tasks/infovqa_suit_VD/_default_template_infovqa_yaml +++ b/lmms_eval/tasks/infovqa_suit_VD/_default_template_infovqa_yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/infovqa +dataset_path: LIME-DATA/infovqa task: "infovqa_suit_val" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/infovqa_suit_VD/infovqa_suit_val.yaml b/lmms_eval/tasks/infovqa_suit_VD/infovqa_suit_val.yaml index bf371d4e..c12a5383 100755 --- a/lmms_eval/tasks/infovqa_suit_VD/infovqa_suit_val.yaml +++ b/lmms_eval/tasks/infovqa_suit_VD/infovqa_suit_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/infovqa +dataset_path: LIME-DATA/infovqa task: "infovqa_suit_val_vd" test_split: train output_type: generate_until diff --git a/lmms_eval/tasks/ocrbench_suit/ocrbench_suit.yaml b/lmms_eval/tasks/ocrbench_suit/ocrbench_suit.yaml index 8c4a95c2..20747299 100644 --- a/lmms_eval/tasks/ocrbench_suit/ocrbench_suit.yaml +++ b/lmms_eval/tasks/ocrbench_suit/ocrbench_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ocrbench +dataset_path: LIME-DATA/ocrbench dataset_kwargs: token: True task: "ocrbench_suit" diff --git a/lmms_eval/tasks/ocrbench_suit_vd/ocrbench_suit.yaml b/lmms_eval/tasks/ocrbench_suit_vd/ocrbench_suit.yaml index cac46dda..77c1f17b 100644 --- a/lmms_eval/tasks/ocrbench_suit_vd/ocrbench_suit.yaml +++ b/lmms_eval/tasks/ocrbench_suit_vd/ocrbench_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/ocrbench +dataset_path: LIME-DATA/ocrbench dataset_kwargs: token: True task: "ocrbench_suit_vd" diff --git a/lmms_eval/tasks/ok_vqa_suit/ok_vqa_val2014.yaml b/lmms_eval/tasks/ok_vqa_suit/ok_vqa_val2014.yaml index bbdc5f87..8e3f5152 100755 --- a/lmms_eval/tasks/ok_vqa_suit/ok_vqa_val2014.yaml +++ b/lmms_eval/tasks/ok_vqa_suit/ok_vqa_val2014.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/OK-VQA +dataset_path: LIME-DATA/OK-VQA group: ok_vqa_suit task: ok_vqa_suit_val2014 test_split: train diff --git a/lmms_eval/tasks/ok_vqa_suit_vd/ok_vqa_val2014.yaml b/lmms_eval/tasks/ok_vqa_suit_vd/ok_vqa_val2014.yaml index 0c3ef524..cdeb3ec8 100755 --- a/lmms_eval/tasks/ok_vqa_suit_vd/ok_vqa_val2014.yaml +++ b/lmms_eval/tasks/ok_vqa_suit_vd/ok_vqa_val2014.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/OK-VQA +dataset_path: LIME-DATA/OK-VQA group: ok_vqa_suit_vd task: ok_vqa_suit_val2014_vd test_split: train diff --git a/lmms_eval/tasks/pope_suit/pope.yaml b/lmms_eval/tasks/pope_suit/pope.yaml index 6b8c0978..1b271ebc 100755 --- a/lmms_eval/tasks/pope_suit/pope.yaml +++ b/lmms_eval/tasks/pope_suit/pope.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/POPE +dataset_path: LIME-DATA/POPE dataset_kwargs: token: True task: "pope_suit" diff --git a/lmms_eval/tasks/pope_suit_vd/pope.yaml b/lmms_eval/tasks/pope_suit_vd/pope.yaml index 692f514f..5702a862 100755 --- a/lmms_eval/tasks/pope_suit_vd/pope.yaml +++ b/lmms_eval/tasks/pope_suit_vd/pope.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/POPE +dataset_path: LIME-DATA/POPE dataset_kwargs: token: True task: "pope_suit_vd" diff --git a/lmms_eval/tasks/scienceqa_suit/scienceqa_img_suit.yaml b/lmms_eval/tasks/scienceqa_suit/scienceqa_img_suit.yaml index 65dd2fe9..a220565f 100755 --- a/lmms_eval/tasks/scienceqa_suit/scienceqa_img_suit.yaml +++ b/lmms_eval/tasks/scienceqa_suit/scienceqa_img_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/scienceqa +dataset_path: LIME-DATA/scienceqa task: "scienceqa_img_suit" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/scienceqa_suit_vd/scienceqa_img_suit.yaml b/lmms_eval/tasks/scienceqa_suit_vd/scienceqa_img_suit.yaml index c66c33cc..d8cd0c78 100755 --- a/lmms_eval/tasks/scienceqa_suit_vd/scienceqa_img_suit.yaml +++ b/lmms_eval/tasks/scienceqa_suit_vd/scienceqa_img_suit.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/scienceqa +dataset_path: LIME-DATA/scienceqa task: "scienceqa_img_suit_vd" dataset_kwargs: token: True diff --git a/lmms_eval/tasks/textcaps_suit/textcaps_suit_val.yaml b/lmms_eval/tasks/textcaps_suit/textcaps_suit_val.yaml index a2504251..4a874c38 100755 --- a/lmms_eval/tasks/textcaps_suit/textcaps_suit_val.yaml +++ b/lmms_eval/tasks/textcaps_suit/textcaps_suit_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/TextCaps +dataset_path: LIME-DATA/TextCaps dataset_kwargs: token: True task: "textcaps_suit_val" diff --git a/lmms_eval/tasks/textcaps_suit_vd/textcaps_suit_val_vd.yaml b/lmms_eval/tasks/textcaps_suit_vd/textcaps_suit_val_vd.yaml index d74aa073..f515ad5f 100755 --- a/lmms_eval/tasks/textcaps_suit_vd/textcaps_suit_val_vd.yaml +++ b/lmms_eval/tasks/textcaps_suit_vd/textcaps_suit_val_vd.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/TextCaps +dataset_path: LIME-DATA/TextCaps dataset_kwargs: token: True task: "textcaps_suit_val_vd" diff --git a/lmms_eval/tasks/textvqa_suit/textvqa_suit_val.yaml b/lmms_eval/tasks/textvqa_suit/textvqa_suit_val.yaml index d77b3eef..a1e1a1ed 100755 --- a/lmms_eval/tasks/textvqa_suit/textvqa_suit_val.yaml +++ b/lmms_eval/tasks/textvqa_suit/textvqa_suit_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/textvqa +dataset_path: LIME-DATA/textvqa task: textvqa_suit_val test_split: train metric_list: diff --git a/lmms_eval/tasks/textvqa_suit_vd/textvqa_suit_val.yaml b/lmms_eval/tasks/textvqa_suit_vd/textvqa_suit_val.yaml index ff9dcea7..04bbd16e 100755 --- a/lmms_eval/tasks/textvqa_suit_vd/textvqa_suit_val.yaml +++ b/lmms_eval/tasks/textvqa_suit_vd/textvqa_suit_val.yaml @@ -1,4 +1,4 @@ -dataset_path: LIME-ICLR/textvqa +dataset_path: LIME-DATA/textvqa task: textvqa_suit_val_vd test_split: train metric_list: