From e5507b5dee4e3c5b452b60b4b86022676f35381f Mon Sep 17 00:00:00 2001 From: Thomas Wolf Date: Thu, 1 Feb 2024 14:28:08 +0000 Subject: [PATCH 1/5] fix --- src/lighteval/metrics/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/lighteval/metrics/__init__.py b/src/lighteval/metrics/__init__.py index 3b17854e7..13a22563a 100644 --- a/src/lighteval/metrics/__init__.py +++ b/src/lighteval/metrics/__init__.py @@ -8,11 +8,12 @@ def apply_target_perplexity_metric(results: list[ModelReturn], formatted_doc: Doc, metrics: list[str]): outputs = {} - current_results = [results.pop(0) for _ in range(len(formatted_doc.get_golds()))] + current_result = results.pop(0) + reference_text = formatted_doc.choices[formatted_doc.gold_index] for metric in metrics: - if Metrics[metric].value.category == MetricCategory.PERPLEXITY: - outputs.update(Metrics[metric].value.compute(results=current_results)) + if Metrics[metric].value.category == MetricCategory.TARGET_PERPLEXITY: + outputs.update(Metrics[metric].value.compute(results=current_result, reference_text=reference_text)) return results, outputs From 6f5075387fe1fc234608c356df53cd252e5c5a92 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 1 Feb 2024 16:37:52 +0000 Subject: [PATCH 2/5] test v1 --- tests/reference_scores/harness_metrics.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/reference_scores/harness_metrics.json b/tests/reference_scores/harness_metrics.json index a6c506f34..965f4584a 100644 --- a/tests/reference_scores/harness_metrics.json +++ b/tests/reference_scores/harness_metrics.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a1965f0b9c66cfe1b1f3cc380a80949e32eab92ae8eac079c0339506ce827093 -size 48373142 +oid sha256:9a6ae69d627f8d0d94da5e871c015acfafa226c68981f6935b511e1f0edf9656 +size 48376617 From febcb7c0d2f742b6bd361004ad1e803808db6510 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 1 Feb 2024 17:51:03 +0000 Subject: [PATCH 3/5] fix target perplexity --- src/lighteval/metrics/__init__.py | 8 +++++--- src/lighteval/metrics/metrics_sample.py | 7 +++---- src/lighteval/metrics/sample_preparator.py | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/lighteval/metrics/__init__.py b/src/lighteval/metrics/__init__.py index 13a22563a..b6ca64919 100644 --- a/src/lighteval/metrics/__init__.py +++ b/src/lighteval/metrics/__init__.py @@ -8,12 +8,14 @@ def apply_target_perplexity_metric(results: list[ModelReturn], formatted_doc: Doc, metrics: list[str]): outputs = {} + reference_text = formatted_doc.get_golds()[0] current_result = results.pop(0) - reference_text = formatted_doc.choices[formatted_doc.gold_index] + target_logprob = current_result.result[0] + target_acc = current_result.result[1] for metric in metrics: if Metrics[metric].value.category == MetricCategory.TARGET_PERPLEXITY: - outputs.update(Metrics[metric].value.compute(results=current_result, reference_text=reference_text)) + outputs.update(Metrics[metric].value.compute(logprobs=target_logprob, target_acc=target_acc, reference_text=reference_text)) return results, outputs @@ -31,7 +33,7 @@ def apply_perplexity_metric(results: list[ModelReturn], formatted_doc: Doc, metr for metric in metrics: if Metrics[metric].value.category == MetricCategory.PERPLEXITY: - outputs.update(Metrics[metric].value.compute(results=current_result, reference_text=reference_text)) + outputs.update(Metrics[metric].value.compute(logprobs=current_result.result, reference_text=reference_text)) return results, outputs diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py index ec123741b..15070d341 100644 --- a/src/lighteval/metrics/metrics_sample.py +++ b/src/lighteval/metrics/metrics_sample.py @@ -275,17 +275,16 @@ def compute(self, choices_logprob: list[float], gold_ixs: list[float], formatted return 1.0 / (min(ranked_choices) + 1) -def acc_golds_likelihood(results: list[tuple[float, int]], **kwargs) -> int: +def acc_golds_likelihood(target_acc: list[int] | int, **kwargs) -> int: """Tests if at least one of predicted gold targets' log-likelihood is above 0.5. Args: - results (list[int]): List of tuples containing, for each gold, the predictions log-probabilities associated with whether they are above 0.5 aggregated. - formatted_doc (Doc): _description_ + target_acc (list[int]): List of scores indicating whether the predictions log-probabilities are above 0.5 aggregated. Returns: int: 1 if at least one of the possible golds had a log-likelihood above 0.5. """ - return max([int(acc_ppl) for _, acc_ppl in results]) + return max([int(acc_ppl) for acc_ppl in as_list(target_acc)]) class ROUGE: diff --git a/src/lighteval/metrics/sample_preparator.py b/src/lighteval/metrics/sample_preparator.py index 659022920..c28ed2470 100644 --- a/src/lighteval/metrics/sample_preparator.py +++ b/src/lighteval/metrics/sample_preparator.py @@ -106,14 +106,14 @@ def count_units(self, text: str) -> int: if self.units_type == "bytes": return len(text.encode("utf-8")) - def prepare(self, results, reference_text, **kwargs): + def prepare(self, logprobs: list[float] | float, reference_text: str, **kwargs): """Prepares an individual perplexity example to the format expected by metrics computed at the corpus level (aggregated). Args: - results (list[float]): List of the logprobabilities computed for each item + logprobs (list[float]): List of the logprobabilities computed for each item of the sequence or single aggregated logprob over the sequence reference_text (str): Current reference text for which to compute the length in self.units_type Returns: PerplexityCorpusMetricInput: Stores the measured logprobs and associated text lengths, counted in the reference unit. """ - return PerplexityCorpusMetricInput(logprobs=results.result, weights=self.count_units(reference_text)) + return PerplexityCorpusMetricInput(logprobs=logprobs, weights=self.count_units(reference_text)) From 3fff1ae27cce5a52dce4ce527e06d9ac7ff99179 Mon Sep 17 00:00:00 2001 From: "clementine@huggingface.co" Date: Thu, 1 Feb 2024 17:56:37 +0000 Subject: [PATCH 4/5] style --- README.md | 10 +++++----- src/lighteval/metrics/__init__.py | 10 ++++++++-- tasks_examples/open_llm_leaderboard_tasks.txt | 2 +- tests/reference_scores/harness_metrics.json | 4 ++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 2e2f35e6d..c04a66118 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ LightEval is an evaluation suite which gathers a selection of features from wide It is still an early, internal version - it should be nice to use but don't expect 100% stability! -In case of problems or question, feel free to open an issue! +In case of problems or question, feel free to open an issue! ## How to install and use ### Requirements @@ -50,11 +50,11 @@ Lastly, create a **line summary** of your evaluation, in `metadata_table.json`. - `suite` (list), the suite(s) to which your evaluation should belong. This field allows us to compare different tasks implementation, and is used a task selection to differentiate the versions to launch. At the moment, you'll find the keywords ["helm", "bigbench", "original", "lighteval"]; you can add also add new ones (for test, we recommend using "custom"). - `prompt_function` (str), the name of the prompt function you defined in the step above - `hf_repo` (str), the path to your evaluation dataset on the hub -- `hf_subset` (str), the specific subset you want to use for your evaluation (note: when the dataset has no subset, fill this field with `"default"`, not with `None` or `""`) +- `hf_subset` (str), the specific subset you want to use for your evaluation (note: when the dataset has no subset, fill this field with `"default"`, not with `None` or `""`) - `hf_avail_splits` (list), all the splits available for your dataset (train, valid or validation, test, other...) - `evaluation_splits` (list), the splits you want to use for evaluation - `few_shots_split` (str, can be `null`), the specific split from which you want to select samples for your few-shot examples. It should be different from the sets included in `evaluation_splits` -- `few_shots_select` (str, can be `null`), the method that you will use to select items for your few-shot examples. Can be `null`, or one of: +- `few_shots_select` (str, can be `null`), the method that you will use to select items for your few-shot examples. Can be `null`, or one of: - `balanced` selects examples from the `few_shots_split` with balanced labels, to avoid skewing the few shot examples (hence the model generations) towards one specific label - `random` selects examples at random from the `few_shots_split` - `random_sampling` selects new examples at random from the `few_shots_split` for every new item, but if a sampled item is equal to the current one, it is removed from the available samples @@ -102,7 +102,7 @@ These metrics need the model to generate an output. They are therefore slower. - `exact_match_indicator`: Exact match with some preceding context (before an indicator) removed - `f1_score_quasi` (HELM): Average F1 score in terms of word overlap between the model output and gold, with both being normalized first - `f1_score`: Average F1 score in terms of word overlap between the model output and gold without normalisation - - `f1_score_macro`: Corpus level macro F1 score + - `f1_score_macro`: Corpus level macro F1 score - `f1_score_macro`: Corpus level micro F1 score - Summarization: - `rouge` (Harness): Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) @@ -141,7 +141,7 @@ These metrics need both the generation and its logprob. They are not working at - `prediction_perplexity` (HELM): Measure of the logprob of a given input. ## Adding a new metric -If you want to add a new metric, first check if you can use one of the parametrized functions in `src.lighteval.metrics.metrics_corpus` or `metrics_sample`. If not, add it to either of these files depending on the level at which it is applied. Then, follow the example in `src.lighteval.metrics.metrics` to register your metric. +If you want to add a new metric, first check if you can use one of the parametrized functions in `src.lighteval.metrics.metrics_corpus` or `metrics_sample`. If not, add it to either of these files depending on the level at which it is applied. Then, follow the example in `src.lighteval.metrics.metrics` to register your metric. ## Examples of scripts to launch lighteval on the cluster ### Evaluate a whole suite on one node, 8 GPUs diff --git a/src/lighteval/metrics/__init__.py b/src/lighteval/metrics/__init__.py index b6ca64919..3a0984bfc 100644 --- a/src/lighteval/metrics/__init__.py +++ b/src/lighteval/metrics/__init__.py @@ -15,7 +15,11 @@ def apply_target_perplexity_metric(results: list[ModelReturn], formatted_doc: Do for metric in metrics: if Metrics[metric].value.category == MetricCategory.TARGET_PERPLEXITY: - outputs.update(Metrics[metric].value.compute(logprobs=target_logprob, target_acc=target_acc, reference_text=reference_text)) + outputs.update( + Metrics[metric].value.compute( + logprobs=target_logprob, target_acc=target_acc, reference_text=reference_text + ) + ) return results, outputs @@ -33,7 +37,9 @@ def apply_perplexity_metric(results: list[ModelReturn], formatted_doc: Doc, metr for metric in metrics: if Metrics[metric].value.category == MetricCategory.PERPLEXITY: - outputs.update(Metrics[metric].value.compute(logprobs=current_result.result, reference_text=reference_text)) + outputs.update( + Metrics[metric].value.compute(logprobs=current_result.result, reference_text=reference_text) + ) return results, outputs diff --git a/tasks_examples/open_llm_leaderboard_tasks.txt b/tasks_examples/open_llm_leaderboard_tasks.txt index 41c0ff35a..5736e9537 100644 --- a/tasks_examples/open_llm_leaderboard_tasks.txt +++ b/tasks_examples/open_llm_leaderboard_tasks.txt @@ -57,4 +57,4 @@ lighteval|mmlu:security_studies|5|0 lighteval|mmlu:sociology|5|0 lighteval|mmlu:us_foreign_policy|5|0 lighteval|mmlu:virology|5|0 -lighteval|mmlu:world_religions|5|0 \ No newline at end of file +lighteval|mmlu:world_religions|5|0 diff --git a/tests/reference_scores/harness_metrics.json b/tests/reference_scores/harness_metrics.json index 965f4584a..1c8c5b91d 100644 --- a/tests/reference_scores/harness_metrics.json +++ b/tests/reference_scores/harness_metrics.json @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9a6ae69d627f8d0d94da5e871c015acfafa226c68981f6935b511e1f0edf9656 -size 48376617 +oid sha256:408956938a6b7a18b03658bb9772b471efcea4aa04afb0b35d76cecfca6a706e +size 48376580 From e1811c614bf8078988c3291bb852cc35dbae6f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Tue, 6 Feb 2024 14:40:50 +0100 Subject: [PATCH 5/5] update after review --- src/lighteval/metrics/metrics_sample.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py index 15070d341..e87e3bb58 100644 --- a/src/lighteval/metrics/metrics_sample.py +++ b/src/lighteval/metrics/metrics_sample.py @@ -1,6 +1,8 @@ """This module manages all the metrics occurring at the sample level. The results of said metrics are then aggregated using simple function (min, mean, max, ...) at the corpus level. Most metrics fall under this category. """ +from typing import Union + import nltk import numpy as np from nltk.metrics.distance import edit_distance @@ -275,7 +277,7 @@ def compute(self, choices_logprob: list[float], gold_ixs: list[float], formatted return 1.0 / (min(ranked_choices) + 1) -def acc_golds_likelihood(target_acc: list[int] | int, **kwargs) -> int: +def acc_golds_likelihood(target_acc: Union[list[int], int], **kwargs) -> int: """Tests if at least one of predicted gold targets' log-likelihood is above 0.5. Args: