Skip to content

Commit

Permalink
qual fix
Browse files Browse the repository at this point in the history
  • Loading branch information
clefourrier committed Jan 26, 2024
1 parent 1100cff commit 109ce8b
Showing 1 changed file with 6 additions and 7 deletions.
13 changes: 6 additions & 7 deletions src/lighteval/metrics/metrics_corpus.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,14 @@ def __init__(self, average: str, num_classes: int = 2):
num_classes (int, optional): Num of possible choice classes. Defaults to 2. If this parameter is above 2, we'll compute multi f1 corpus score
"""
if self.average not in ["weighted", "macro", "micro"]:
raise ValueError(f"A CorpusLevelF1Score must be initialized with weighted, macro, micro as an average function. {average} was used.")
raise ValueError(
f"A CorpusLevelF1Score must be initialized with weighted, macro, micro as an average function. {average} was used."
)
self.average = average
self.num_classes = num_classes

def compute(self, items: list[LogprobCorpusMetricInput]):
"""Computes the metric score over all the corpus generated items, by using the scikit learn implementation.
"""
"""Computes the metric score over all the corpus generated items, by using the scikit learn implementation."""
golds = [i.golds for i in items]
preds = [i.preds for i in items]
# Single f1
Expand Down Expand Up @@ -78,8 +79,7 @@ def __init__(self, metric_type: str):
raise ValueError(f"Unknown corpus level translation metric type : {metric_type}")

def compute(self, items: list[GenerativeCorpusMetricInput]) -> float:
"""Computes the metric score over all the corpus generated items, by using the sacrebleu implementation.
"""
"""Computes the metric score over all the corpus generated items, by using the sacrebleu implementation."""
golds = [i.golds for i in items]
preds = [as_list(i.preds) for i in items]
return float(self.metric(hypotheses=preds, references=golds).score)
Expand All @@ -104,8 +104,7 @@ def __init__(self, metric_type: str):
self.metric_type = metric_type

def compute(self, items: list[PerplexityCorpusMetricInput]):
"""Computes the metric score over all the corpus generated items.
"""
"""Computes the metric score over all the corpus generated items."""
logprobs = [i.logprobs for i in items]
weights = [i.weights for i in items]

Expand Down

0 comments on commit 109ce8b

Please sign in to comment.