From 81be45c5be0243375becdfe489bebee7e2a63b94 Mon Sep 17 00:00:00 2001 From: alexnicita Date: Wed, 31 Jul 2024 17:57:13 -0400 Subject: [PATCH] reorganize data folder (news) --- ai/llm/executor.py | 10 +- ai/rag/chroma.py | 11 +- ai/utils/utils.py | 5 - api/polymarket.py | 150 +++++++++--------- .../newsapiorg/api.py} | 10 +- .../newsapi_org => news}/types.py | 0 data/newspaper.py | 7 + .../tavily-demo.py => search/tavily.py} | 0 scripts/python/cli.py | 2 +- 9 files changed, 91 insertions(+), 104 deletions(-) rename data/{news_providers/newsapi_org/newsapi_caller.py => news/newsapiorg/api.py} (93%) rename data/{news_providers/newsapi_org => news}/types.py (100%) rename data/{web_search/tavily-demo.py => search/tavily.py} (100%) diff --git a/ai/llm/executor.py b/ai/llm/executor.py index 382f443..fab756c 100644 --- a/ai/llm/executor.py +++ b/ai/llm/executor.py @@ -5,7 +5,7 @@ from langchain_openai import ChatOpenAI from ai.llm import prompts -from api.polymarket.gamma import GammaMarketClient +from api.gamma import GammaMarketClient from polymarket.agents.ai.llm.prompts import Prompter @@ -18,6 +18,7 @@ def __init__(self): model="gpt-3.5-turbo", temperature=0, ) + self.client = GammaMarketClient() def get_llm_response(self, user_input: str) -> str: system_message = SystemMessage(content=str(prompts.market_analyst)) @@ -36,15 +37,14 @@ def get_superforecast( return result.content def get_polymarket_llm(self, user_input: str) -> str: - client = GammaMarketClient() - data1 = client.get_current_events() - data2 = client.get_current_markets() + data1 = self.client.get_current_events() + data2 = self.client.get_current_markets() system_message = SystemMessage( content=str(prompts.prompts_polymarket(data1=data1, data2=data2)) ) human_message = HumanMessage(content=user_input) messages = [system_message, human_message] - result = llm.invoke(messages) + result = self.llm.invoke(messages) return result.content def filter_events(self): diff --git a/ai/rag/chroma.py b/ai/rag/chroma.py index 415b042..cbc996c 100644 --- a/ai/rag/chroma.py +++ b/ai/rag/chroma.py @@ -1,11 +1,11 @@ import json import os import time -from devtools import pprint + from langchain_chroma import Chroma from langchain_openai import OpenAIEmbeddings from langchain_community.document_loaders import JSONLoader -from api.polymarket.gamma import GammaMarketClient +from api.gamma import GammaMarketClient class PolymarketRAG: @@ -51,10 +51,3 @@ def query_local_markets_rag(self, local_directory=None, query=None): ) response_docs = local_db.similarity_search_with_score(query=query) return response_docs - - -# TODO: -# 1. Pull available markets -# 2. Prompt to find a market to trade on -# 3. Prompt to find which side of the market to place a bet on -# 4. Execute the trade on that market diff --git a/ai/utils/utils.py b/ai/utils/utils.py index 94b5236..4000680 100644 --- a/ai/utils/utils.py +++ b/ai/utils/utils.py @@ -1,7 +1,6 @@ import json -# TODO: this could be done using regex for sleeker but less readable code. def parse_camel_case(key): output = "" for char in key: @@ -13,7 +12,6 @@ def parse_camel_case(key): return output -# Experimenting with converting json fields into natural language to improve retrieval from text embed mode def preprocess_market_object(market_object): description = market_object["description"] @@ -49,9 +47,6 @@ def preprocess_local_json(file_path, preprocessor_function): json.dump(output, output_file) -# Options for improving search: -# 1. Translate JSON params into natural language -# 2. Metadata function with post-filtering on metadata kv pairs def metadata_func(record: dict, metadata: dict) -> dict: print("record:", record) print("meta:", metadata) diff --git a/api/polymarket.py b/api/polymarket.py index dc177df..3dace20 100644 --- a/api/polymarket.py +++ b/api/polymarket.py @@ -7,7 +7,7 @@ import ast from dotenv import load_dotenv -from pprint import pprint + from web3 import Web3 from web3.constants import MAX_INT from web3.middleware import geth_poa_middleware @@ -28,82 +28,6 @@ load_dotenv() -def auth(): - pass - - -def test(): - host = "https://clob.polymarket.com" - key = os.getenv("POLYGON_WALLET_PRIVATE_KEY") - print(key) - chain_id = POLYGON - - # Create CLOB client and get/set API credentials - client = ClobClient(host, key=key, chain_id=chain_id) - client.set_api_creds(client.create_or_derive_api_creds()) - - creds = ApiCreds( - api_key=os.getenv("CLOB_API_KEY"), - api_secret=os.getenv("CLOB_SECRET"), - api_passphrase=os.getenv("CLOB_PASS_PHRASE"), - ) # TODO: post /auth/api-key - chain_id = AMOY - client = ClobClient(host, key=key, chain_id=chain_id, creds=creds) - - print(client.get_markets()) - print(client.get_simplified_markets()) - print(client.get_sampling_markets()) - print(client.get_sampling_simplified_markets()) - print(client.get_market("condition_id")) - - print("Done!") - - -def gamma(): - url = "https://gamma-api.polymarket.com" - markets_url = url + "/markets" - res = httpx.get(markets_url) - code = res.status_code - if code == 200: - markets: list[SimpleMarket] = [] - data = res.json() - for market in data: - try: - market_data = { - "id": int(market["id"]), - "question": market["question"], - # "start": market['startDate'], - "end": market["endDate"], - "description": market["description"], - "active": market["active"], - "deployed": market["deployed"], - "funded": market["funded"], - # "orderMinSize": float(market['orderMinSize']) if market['orderMinSize'] else 0, - # "orderPriceMinTickSize": float(market['orderPriceMinTickSize']), - "rewardsMinSize": float(market["rewardsMinSize"]), - "rewardsMaxSpread": float(market["rewardsMaxSpread"]), - "volume": float(market["volume"]), - "spread": float(market["spread"]), - "outcome_a": str(market["outcomes"][0]), - "outcome_b": str(market["outcomes"][1]), - "outcome_a_price": str(market["outcomePrices"][0]), - "outcome_b_price": str(market["outcomePrices"][1]), - } - markets.append(SimpleMarket(**market_data)) - except Exception as err: - print(f"error {err} for market {id}") - pdb.set_trace() - else: - raise Exception() - - -def main(): - # auth() - # test() - # gamma() - print(Polymarket().get_all_events()) - - class Polymarket: def __init__(self): self.gamma_url = "https://gamma-api.polymarket.com" @@ -394,6 +318,78 @@ def execute_order(self, price, size, side, token_id): ) +def test(): + host = "https://clob.polymarket.com" + key = os.getenv("POLYGON_WALLET_PRIVATE_KEY") + print(key) + chain_id = POLYGON + + # Create CLOB client and get/set API credentials + client = ClobClient(host, key=key, chain_id=chain_id) + client.set_api_creds(client.create_or_derive_api_creds()) + + creds = ApiCreds( + api_key=os.getenv("CLOB_API_KEY"), + api_secret=os.getenv("CLOB_SECRET"), + api_passphrase=os.getenv("CLOB_PASS_PHRASE"), + ) # TODO: post /auth/api-key + chain_id = AMOY + client = ClobClient(host, key=key, chain_id=chain_id, creds=creds) + + print(client.get_markets()) + print(client.get_simplified_markets()) + print(client.get_sampling_markets()) + print(client.get_sampling_simplified_markets()) + print(client.get_market("condition_id")) + + print("Done!") + + +def gamma(): + url = "https://gamma-api.polymarket.com" + markets_url = url + "/markets" + res = httpx.get(markets_url) + code = res.status_code + if code == 200: + markets: list[SimpleMarket] = [] + data = res.json() + for market in data: + try: + market_data = { + "id": int(market["id"]), + "question": market["question"], + # "start": market['startDate'], + "end": market["endDate"], + "description": market["description"], + "active": market["active"], + "deployed": market["deployed"], + "funded": market["funded"], + # "orderMinSize": float(market['orderMinSize']) if market['orderMinSize'] else 0, + # "orderPriceMinTickSize": float(market['orderPriceMinTickSize']), + "rewardsMinSize": float(market["rewardsMinSize"]), + "rewardsMaxSpread": float(market["rewardsMaxSpread"]), + "volume": float(market["volume"]), + "spread": float(market["spread"]), + "outcome_a": str(market["outcomes"][0]), + "outcome_b": str(market["outcomes"][1]), + "outcome_a_price": str(market["outcomePrices"][0]), + "outcome_b_price": str(market["outcomePrices"][1]), + } + markets.append(SimpleMarket(**market_data)) + except Exception as err: + print(f"error {err} for market {id}") + pdb.set_trace() + else: + raise Exception() + + +def main(): + # auth() + # test() + # gamma() + print(Polymarket().get_all_events()) + + if __name__ == "__main__": load_dotenv() diff --git a/data/news_providers/newsapi_org/newsapi_caller.py b/data/news/newsapiorg/api.py similarity index 93% rename from data/news_providers/newsapi_org/newsapi_caller.py rename to data/news/newsapiorg/api.py index dc33586..c89957c 100644 --- a/data/news_providers/newsapi_org/newsapi_caller.py +++ b/data/news/newsapiorg/api.py @@ -1,15 +1,11 @@ -import json import os -import pathlib -from dotenv import load_dotenv -from newsapi import NewsApiClient -from data_sources.news_providers.newsapi_org.types import Article +from newsapi import NewsApiClient -load_dotenv() +from data.news.types import Article -class NewsApiCaller: +class API: def __init__(self) -> None: self.configs = { "language": "en", diff --git a/data/news_providers/newsapi_org/types.py b/data/news/types.py similarity index 100% rename from data/news_providers/newsapi_org/types.py rename to data/news/types.py diff --git a/data/newspaper.py b/data/newspaper.py index 580cb4d..8fb576d 100644 --- a/data/newspaper.py +++ b/data/newspaper.py @@ -1,2 +1,9 @@ +""" + +Hybrid class to bridge many news data sources. + +""" + + class Newspaper: pass diff --git a/data/web_search/tavily-demo.py b/data/search/tavily.py similarity index 100% rename from data/web_search/tavily-demo.py rename to data/search/tavily.py diff --git a/scripts/python/cli.py b/scripts/python/cli.py index 3c9e48d..e627c16 100644 --- a/scripts/python/cli.py +++ b/scripts/python/cli.py @@ -4,7 +4,7 @@ from api.polymarket import Polymarket from ai.llm import executor, prompts from ai.rag.chroma import PolymarketRAG -from data.news_providers.newsapi_org.newsapi_caller import NewsApiCaller +from data.news.newsapiorg.api import API as NewsApiCaller from jobs.scheduler import TradingAgent app = typer.Typer()