diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index 62e65a1eb8..8db5ee31bb 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -45,6 +45,7 @@ | [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | | [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French| | [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | +| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | | [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | | [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | | [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | diff --git a/lm_eval/tasks/global_mmlu/README.md b/lm_eval/tasks/global_mmlu/README.md new file mode 100644 index 0000000000..838a7c9d42 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/README.md @@ -0,0 +1,25 @@ +# Global-MMLU + +### Paper + +Title: `Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation` + +Abstract: [https://arxiv.org/abs/2412.03304](https://arxiv.org/abs/2412.03304) + +Global-MMLU-Lite is a balanced collection of culturally sensitive and culturally agnostic MMLU tasks. It is designed for efficient evaluation of multilingual models in 15 languages (including English). Only languages with human translations and post-edits in the original [Global-MMLU](https://huggingface.co/datasets/CohereForAI/Global-MMLU) 🌍 dataset have been included in the lite version. + +Homepage: [https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite](https://huggingface.co/datasets/CohereForAI/Global-MMLU-Lite) + +### Citation + +```bibtex +@misc{singh2024globalmmluunderstandingaddressing, + title={Global MMLU: Understanding and Addressing Cultural and Linguistic Biases in Multilingual Evaluation}, + author={Shivalika Singh and Angelika Romanou and Clémentine Fourrier and David I. Adelani and Jian Gang Ngui and Daniel Vila-Suero and Peerat Limkonchotiwat and Kelly Marchisio and Wei Qi Leong and Yosephine Susanto and Raymond Ng and Shayne Longpre and Wei-Yin Ko and Madeline Smith and Antoine Bosselut and Alice Oh and Andre F. T. Martins and Leshem Choshen and Daphne Ippolito and Enzo Ferrante and Marzieh Fadaee and Beyza Ermis and Sara Hooker}, + year={2024}, + eprint={2412.03304}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2412.03304}, +} +``` diff --git a/lm_eval/tasks/global_mmlu/_default_yaml b/lm_eval/tasks/global_mmlu/_default_yaml new file mode 100644 index 0000000000..33a1fc356a --- /dev/null +++ b/lm_eval/tasks/global_mmlu/_default_yaml @@ -0,0 +1,17 @@ +tag: + - global_mmlu +dataset_path: CohereForAI/Global-MMLU-Lite +test_split: test +fewshot_split: dev +fewshot_config: + sampler: default +output_type: multiple_choice +doc_to_text: "{{question.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/global_mmlu/_generate_configs.py b/lm_eval/tasks/global_mmlu/_generate_configs.py new file mode 100644 index 0000000000..58e169c6d4 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/_generate_configs.py @@ -0,0 +1,42 @@ +import yaml + + +languages = [ + "en", + "ar", + "fr", + "es", + "hi", + "de", + "id", + "it", + "ja", + "ko", + "pt", + "zh", + "yo", + "bn", + "sw", +] + + +def main() -> None: + for language in languages: + file_name = f"global_mmlu_{language}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_default_yaml", + "task": f"global_mmlu_{language}", + "dataset_name": language, + }, + f, + ) + except FileExistsError: + pass + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_ar.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_ar.yaml new file mode 100644 index 0000000000..703f420a52 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_ar.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ar +include: _default_yaml +task: global_mmlu_ar diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_bn.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_bn.yaml new file mode 100644 index 0000000000..f85b67a293 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_bn.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: bn +include: _default_yaml +task: global_mmlu_bn diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_de.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_de.yaml new file mode 100644 index 0000000000..a874c64fd5 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_de.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: de +include: _default_yaml +task: global_mmlu_de diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_en.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_en.yaml new file mode 100644 index 0000000000..34a6d7120a --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_en.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: en +include: _default_yaml +task: global_mmlu_en diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_es.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_es.yaml new file mode 100644 index 0000000000..75abc77567 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_es.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: es +include: _default_yaml +task: global_mmlu_es diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_fr.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_fr.yaml new file mode 100644 index 0000000000..1a66f53648 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_fr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fr +include: _default_yaml +task: global_mmlu_fr diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_hi.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_hi.yaml new file mode 100644 index 0000000000..788f95f2d2 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_hi.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hi +include: _default_yaml +task: global_mmlu_hi diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_id.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_id.yaml new file mode 100644 index 0000000000..f4b6d5071d --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_id.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: id +include: _default_yaml +task: global_mmlu_id diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_it.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_it.yaml new file mode 100644 index 0000000000..5b55df975f --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_it.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: it +include: _default_yaml +task: global_mmlu_it diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_ja.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_ja.yaml new file mode 100644 index 0000000000..97d9c6ca48 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_ja.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ja +include: _default_yaml +task: global_mmlu_ja diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_ko.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_ko.yaml new file mode 100644 index 0000000000..02b7fe0388 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_ko.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ko +include: _default_yaml +task: global_mmlu_ko diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_pt.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_pt.yaml new file mode 100644 index 0000000000..724bfb4d23 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_pt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: pt +include: _default_yaml +task: global_mmlu_pt diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_sw.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_sw.yaml new file mode 100644 index 0000000000..481232fa28 --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_sw.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sw +include: _default_yaml +task: global_mmlu_sw diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_yo.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_yo.yaml new file mode 100644 index 0000000000..c6ec2f9efc --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_yo.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: yo +include: _default_yaml +task: global_mmlu_yo diff --git a/lm_eval/tasks/global_mmlu/global_mmlu_zh.yaml b/lm_eval/tasks/global_mmlu/global_mmlu_zh.yaml new file mode 100644 index 0000000000..862d46ad9d --- /dev/null +++ b/lm_eval/tasks/global_mmlu/global_mmlu_zh.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: zh +include: _default_yaml +task: global_mmlu_zh