Skip to content

Commit

Permalink
feat: create separate table for easiest and hardest questions
Browse files Browse the repository at this point in the history
  • Loading branch information
vprelovac committed Oct 18, 2024
1 parent d4d3022 commit 86c3df1
Showing 1 changed file with 9 additions and 14 deletions.
23 changes: 9 additions & 14 deletions llms/llms.py
Original file line number Diff line number Diff line change
Expand Up @@ -773,28 +773,23 @@ def process_prompts_sequentially(model, prompts, evaluator, **kwargs):
elif all_incorrect:
hardest_questions.append((i, problem[0]))

# Create tables for easiest and hardest questions
easy_table = PrettyTable(["Index", "Question"])
hard_table = PrettyTable(["Index", "Question"])
# Create a new table for easiest and hardest questions
questions_table = PrettyTable(["Category", "Index", "Question"])
questions_table.align["Question"] = "l" # Left-align the Question column

for index, question in easiest_questions:
easy_table.add_row([index, question[:100] + ('...' if len(question) > 100 else '')])
questions_table.add_row(["Easiest", index, question[:100] + ('...' if len(question) > 100 else '')])

for index, question in hardest_questions:
hard_table.add_row([index, question[:100] + ('...' if len(question) > 100 else '')])
questions_table.add_row(["Hardest", index, question[:100] + ('...' if len(question) > 100 else '')])

# Add these tables to the main table
table.add_row([""] * len(headers))
table.add_row(["Easiest Questions (All models correct)"] + [""] * (len(headers) - 1))
table.add_row([easy_table.get_string()] + [""] * (len(headers) - 1))
table.add_row([""] * len(headers))
table.add_row(["Hardest Questions (No model correct)"] + [""] * (len(headers) - 1))
table.add_row([hard_table.get_string()] + [""] * (len(headers) - 1))
# Return both tables
return table, questions_table

if not html:
return table
return table, questions_table
else:
return table.get_html_string()
return table.get_html_string(), questions_table.get_html_string()
def _load_api_keys(self, kwargs: Dict[str, Any]) -> None:
self._provider_map = {
name: Provider(
Expand Down

0 comments on commit 86c3df1

Please sign in to comment.