diff --git a/backend/src/lm/batch/batch.py b/backend/src/lm/batch/batch.py deleted file mode 100644 index ce976c11..00000000 --- a/backend/src/lm/batch/batch.py +++ /dev/null @@ -1,36 +0,0 @@ -# from src.lm.prompts import SYSPROMPT -# from typing import List - - -# def generate_batch_prompts() -> List[str]: -# articles = get_articles() -# batch_prompts = [] -# id = 0 -# for article in articles: -# request_id = f"request_{id}" -# id += 1 -# batch_prompts.append( -# { -# "custom_id": request_id, -# "method": "POST", -# "url": "/v1/chat/completions", -# "body": { -# "model": "gpt-4o-mini", -# "messages": [ -# {"role": "system", "content": SYSPROMPT}, -# { -# "role": "user", -# "content": article.get("fields").get("bodyText"), -# }, -# ], -# # "metadata": { -# # "article_id": f"article_{id}", -# # }, -# "max_tokens": 1000, -# }, -# } -# ) - -# with open("batch_prompts.jsonl", "w") as jsonl_file: -# for item in batch_prompts: -# jsonl_file.write(json.dumps(item) + "\n") diff --git a/backend/src/lm/generate_events.py b/backend/src/lm/generate_events.py index 04bd7507..c6e081a3 100644 --- a/backend/src/lm/generate_events.py +++ b/backend/src/lm/generate_events.py @@ -5,7 +5,6 @@ from src.scrapers.guardian.get_articles import get_articles from typing import List from pydantic import BaseModel -from src.scrapers.guardian.scrape import query_page from src.common.constants import LANGCHAIN_API_KEY from src.common.constants import LANGCHAIN_TRACING_V2 from src.common.constants import OPENAI_API_KEY diff --git a/backend/src/scrapers/guardian/scrape.py b/backend/src/scrapers/guardian/scrape.py index 7a24ea66..0704b21d 100644 --- a/backend/src/scrapers/guardian/scrape.py +++ b/backend/src/scrapers/guardian/scrape.py @@ -1,7 +1,4 @@ import httpx -import json -import argparse -import time from src.common.constants import GUARDIAN_API_KEY