reorganize data folder (news)

ai/llm/executor.py

@@ -5,7 +5,7 @@
 from langchain_openai import ChatOpenAI
 
 from ai.llm import prompts
-from api.polymarket.gamma import GammaMarketClient
+from api.gamma import GammaMarketClient
 from polymarket.agents.ai.llm.prompts import Prompter
 
 
@@ -18,6 +18,7 @@ def __init__(self):
             model="gpt-3.5-turbo",
             temperature=0,
         )
+        self.client = GammaMarketClient()
 
     def get_llm_response(self, user_input: str) -> str:
         system_message = SystemMessage(content=str(prompts.market_analyst))
@@ -36,15 +37,14 @@ def get_superforecast(
         return result.content
 
     def get_polymarket_llm(self, user_input: str) -> str:
-        client = GammaMarketClient()
-        data1 = client.get_current_events()
-        data2 = client.get_current_markets()
+        data1 = self.client.get_current_events()
+        data2 = self.client.get_current_markets()
         system_message = SystemMessage(
             content=str(prompts.prompts_polymarket(data1=data1, data2=data2))
         )
         human_message = HumanMessage(content=user_input)
         messages = [system_message, human_message]
-        result = llm.invoke(messages)
+        result = self.llm.invoke(messages)
         return result.content
 
     def filter_events(self):

ai/rag/chroma.py

@@ -1,11 +1,11 @@
 import json
 import os
 import time
-from devtools import pprint
+
 from langchain_chroma import Chroma
 from langchain_openai import OpenAIEmbeddings
 from langchain_community.document_loaders import JSONLoader
-from api.polymarket.gamma import GammaMarketClient
+from api.gamma import GammaMarketClient
 
 
 class PolymarketRAG:
@@ -51,10 +51,3 @@ def query_local_markets_rag(self, local_directory=None, query=None):
         )
         response_docs = local_db.similarity_search_with_score(query=query)
         return response_docs
-
-
-# TODO:
-# 1. Pull available markets
-# 2. Prompt to find a market to trade on
-# 3. Prompt to find which side of the market to place a bet on
-# 4. Execute the trade on that market

ai/utils/utils.py

@@ -1,7 +1,6 @@
 import json
 
 
-# TODO: this could be done using regex for sleeker but less readable code.
 def parse_camel_case(key):
     output = ""
     for char in key:
@@ -13,7 +12,6 @@ def parse_camel_case(key):
     return output
 
 
-# Experimenting with converting json fields into natural language to improve retrieval from text embed mode
 def preprocess_market_object(market_object):
     description = market_object["description"]
 
@@ -49,9 +47,6 @@ def preprocess_local_json(file_path, preprocessor_function):
         json.dump(output, output_file)
 
 
-# Options for improving search:
-# 1. Translate JSON params into natural language
-# 2. Metadata function with post-filtering on metadata kv pairs
 def metadata_func(record: dict, metadata: dict) -> dict:
     print("record:", record)
     print("meta:", metadata)

api/polymarket.py

@@ -7,7 +7,7 @@
 import ast
 
 from dotenv import load_dotenv
-from pprint import pprint
+
 from web3 import Web3
 from web3.constants import MAX_INT
 from web3.middleware import geth_poa_middleware
@@ -28,82 +28,6 @@
 load_dotenv()
 
 
-def auth():
-    pass
-
-
-def test():
-    host = "https://clob.polymarket.com"
-    key = os.getenv("POLYGON_WALLET_PRIVATE_KEY")
-    print(key)
-    chain_id = POLYGON
-
-    # Create CLOB client and get/set API credentials
-    client = ClobClient(host, key=key, chain_id=chain_id)
-    client.set_api_creds(client.create_or_derive_api_creds())
-
-    creds = ApiCreds(
-        api_key=os.getenv("CLOB_API_KEY"),
-        api_secret=os.getenv("CLOB_SECRET"),
-        api_passphrase=os.getenv("CLOB_PASS_PHRASE"),
-    )  # TODO: post /auth/api-key
-    chain_id = AMOY
-    client = ClobClient(host, key=key, chain_id=chain_id, creds=creds)
-
-    print(client.get_markets())
-    print(client.get_simplified_markets())
-    print(client.get_sampling_markets())
-    print(client.get_sampling_simplified_markets())
-    print(client.get_market("condition_id"))
-
-    print("Done!")
-
-
-def gamma():
-    url = "https://gamma-api.polymarket.com"
-    markets_url = url + "/markets"
-    res = httpx.get(markets_url)
-    code = res.status_code
-    if code == 200:
-        markets: list[SimpleMarket] = []
-        data = res.json()
-        for market in data:
-            try:
-                market_data = {
-                    "id": int(market["id"]),
-                    "question": market["question"],
-                    # "start": market['startDate'],
-                    "end": market["endDate"],
-                    "description": market["description"],
-                    "active": market["active"],
-                    "deployed": market["deployed"],
-                    "funded": market["funded"],
-                    # "orderMinSize": float(market['orderMinSize']) if market['orderMinSize'] else 0,
-                    # "orderPriceMinTickSize": float(market['orderPriceMinTickSize']),
-                    "rewardsMinSize": float(market["rewardsMinSize"]),
-                    "rewardsMaxSpread": float(market["rewardsMaxSpread"]),
-                    "volume": float(market["volume"]),
-                    "spread": float(market["spread"]),
-                    "outcome_a": str(market["outcomes"][0]),
-                    "outcome_b": str(market["outcomes"][1]),
-                    "outcome_a_price": str(market["outcomePrices"][0]),
-                    "outcome_b_price": str(market["outcomePrices"][1]),
-                }
-                markets.append(SimpleMarket(**market_data))
-            except Exception as err:
-                print(f"error {err} for market {id}")
-        pdb.set_trace()
-    else:
-        raise Exception()
-
-
-def main():
-    # auth()
-    # test()
-    # gamma()
-    print(Polymarket().get_all_events())
-
-
 class Polymarket:
     def __init__(self):
         self.gamma_url = "https://gamma-api.polymarket.com"
@@ -394,6 +318,78 @@ def execute_order(self, price, size, side, token_id):
         )
 
 
+def test():
+    host = "https://clob.polymarket.com"
+    key = os.getenv("POLYGON_WALLET_PRIVATE_KEY")
+    print(key)
+    chain_id = POLYGON
+
+    # Create CLOB client and get/set API credentials
+    client = ClobClient(host, key=key, chain_id=chain_id)
+    client.set_api_creds(client.create_or_derive_api_creds())
+
+    creds = ApiCreds(
+        api_key=os.getenv("CLOB_API_KEY"),
+        api_secret=os.getenv("CLOB_SECRET"),
+        api_passphrase=os.getenv("CLOB_PASS_PHRASE"),
+    )  # TODO: post /auth/api-key
+    chain_id = AMOY
+    client = ClobClient(host, key=key, chain_id=chain_id, creds=creds)
+
+    print(client.get_markets())
+    print(client.get_simplified_markets())
+    print(client.get_sampling_markets())
+    print(client.get_sampling_simplified_markets())
+    print(client.get_market("condition_id"))
+
+    print("Done!")
+
+
+def gamma():
+    url = "https://gamma-api.polymarket.com"
+    markets_url = url + "/markets"
+    res = httpx.get(markets_url)
+    code = res.status_code
+    if code == 200:
+        markets: list[SimpleMarket] = []
+        data = res.json()
+        for market in data:
+            try:
+                market_data = {
+                    "id": int(market["id"]),
+                    "question": market["question"],
+                    # "start": market['startDate'],
+                    "end": market["endDate"],
+                    "description": market["description"],
+                    "active": market["active"],
+                    "deployed": market["deployed"],
+                    "funded": market["funded"],
+                    # "orderMinSize": float(market['orderMinSize']) if market['orderMinSize'] else 0,
+                    # "orderPriceMinTickSize": float(market['orderPriceMinTickSize']),
+                    "rewardsMinSize": float(market["rewardsMinSize"]),
+                    "rewardsMaxSpread": float(market["rewardsMaxSpread"]),
+                    "volume": float(market["volume"]),
+                    "spread": float(market["spread"]),
+                    "outcome_a": str(market["outcomes"][0]),
+                    "outcome_b": str(market["outcomes"][1]),
+                    "outcome_a_price": str(market["outcomePrices"][0]),
+                    "outcome_b_price": str(market["outcomePrices"][1]),
+                }
+                markets.append(SimpleMarket(**market_data))
+            except Exception as err:
+                print(f"error {err} for market {id}")
+        pdb.set_trace()
+    else:
+        raise Exception()
+
+
+def main():
+    # auth()
+    # test()
+    # gamma()
+    print(Polymarket().get_all_events())
+
+
 if __name__ == "__main__":
     load_dotenv()
 

data/news_providers/newsapi_org/newsapi_caller.py → data/news/newsapiorg/api.py

@@ -1,15 +1,11 @@
-import json
 import os
-import pathlib
-from dotenv import load_dotenv
-from newsapi import NewsApiClient
 
-from data_sources.news_providers.newsapi_org.types import Article
+from newsapi import NewsApiClient
 
-load_dotenv()
+from data.news.types import Article
 
 
-class NewsApiCaller:
+class API:
     def __init__(self) -> None:
         self.configs = {
             "language": "en",

data/newspaper.py

@@ -1,2 +1,9 @@
+"""
+
+Hybrid class to bridge many news data sources.
+
+"""
+
+
 class Newspaper:
     pass

scripts/python/cli.py

@@ -4,7 +4,7 @@
 from api.polymarket import Polymarket
 from ai.llm import executor, prompts
 from ai.rag.chroma import PolymarketRAG
-from data.news_providers.newsapi_org.newsapi_caller import NewsApiCaller
+from data.news.newsapiorg.api import API as NewsApiCaller
 from jobs.scheduler import TradingAgent
 
 app = typer.Typer()