From 86c3df12e9e1c80e6b36bca1543e9031e49d3ada Mon Sep 17 00:00:00 2001 From: "Vladimir Prelovac (aider)" Date: Thu, 17 Oct 2024 22:46:41 -0700 Subject: [PATCH] feat: create separate table for easiest and hardest questions --- llms/llms.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/llms/llms.py b/llms/llms.py index ea8bb90..248543d 100644 --- a/llms/llms.py +++ b/llms/llms.py @@ -773,28 +773,23 @@ def process_prompts_sequentially(model, prompts, evaluator, **kwargs): elif all_incorrect: hardest_questions.append((i, problem[0])) - # Create tables for easiest and hardest questions - easy_table = PrettyTable(["Index", "Question"]) - hard_table = PrettyTable(["Index", "Question"]) + # Create a new table for easiest and hardest questions + questions_table = PrettyTable(["Category", "Index", "Question"]) + questions_table.align["Question"] = "l" # Left-align the Question column for index, question in easiest_questions: - easy_table.add_row([index, question[:100] + ('...' if len(question) > 100 else '')]) + questions_table.add_row(["Easiest", index, question[:100] + ('...' if len(question) > 100 else '')]) for index, question in hardest_questions: - hard_table.add_row([index, question[:100] + ('...' if len(question) > 100 else '')]) + questions_table.add_row(["Hardest", index, question[:100] + ('...' if len(question) > 100 else '')]) - # Add these tables to the main table - table.add_row([""] * len(headers)) - table.add_row(["Easiest Questions (All models correct)"] + [""] * (len(headers) - 1)) - table.add_row([easy_table.get_string()] + [""] * (len(headers) - 1)) - table.add_row([""] * len(headers)) - table.add_row(["Hardest Questions (No model correct)"] + [""] * (len(headers) - 1)) - table.add_row([hard_table.get_string()] + [""] * (len(headers) - 1)) + # Return both tables + return table, questions_table if not html: - return table + return table, questions_table else: - return table.get_html_string() + return table.get_html_string(), questions_table.get_html_string() def _load_api_keys(self, kwargs: Dict[str, Any]) -> None: self._provider_map = { name: Provider(