From 675dae5a2604fcf1750c3ccf2e6e256f88c356f1 Mon Sep 17 00:00:00 2001 From: Luca Soldaini Date: Sat, 30 Sep 2023 21:20:01 -0700 Subject: [PATCH] switched to loggers instead of print statement --- llms/llms.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/llms/llms.py b/llms/llms.py index e447aed..6790375 100644 --- a/llms/llms.py +++ b/llms/llms.py @@ -17,6 +17,10 @@ import concurrent.futures from concurrent.futures import ThreadPoolExecutor, as_completed from typing import List, Optional, Tuple, Type, Union +from logging import getLogger + + +LOGGER = getLogger(__name__) @dataclass @@ -46,6 +50,7 @@ def __init__(self, **kwargs ): """Programmatically load api keys and instantiate providers.""" + for provider in [p for p in self._possible_providers if p.api_key_name]: assert provider.api_key_name # for static type checking only api_key = None @@ -65,7 +70,7 @@ def __init__(self, for single_model in self._models: for provider in self._possible_providers: if single_model in provider.provider.MODEL_INFO: - print(f"Found {single_model} in {provider.provider.__name__}") + LOGGER.info(f"Found {single_model} in {provider.provider.__name__}") if provider.api_key: self._providers.append(provider.provider(api_key=provider.api_key, model=single_model)) elif not provider.needs_api_key: @@ -217,7 +222,7 @@ def benchmark(self, problems=None, evaluator=None, show_outputs=False, html=Fals Email: first.name.wild🐻@email.com\ Phone: 5551112222\ ", - ), + ), ("Please count the number of t in eeooeotetto", "3"), ( "Use m to substitute p, a to substitute e, n to substitute a, g to substitute c, o to substitute h,\ @@ -269,7 +274,7 @@ def benchmark(self, problems=None, evaluator=None, show_outputs=False, html=Fals Question: Is there a series of flights that goes from city F to city I?", "No"), ('Bob (a boy) has 3 sisters. Each sister has 2 brothers. How many brothers does Bob have?', '1') ] - + def evaluate_answers( evaluator, query_answer_pairs: List[Tuple[str, str]]