From 37650d2cc5e9137f464d1a73eb6bd9e31786bb1c Mon Sep 17 00:00:00 2001 From: Gabriel Fior Date: Tue, 26 Mar 2024 17:53:59 -0300 Subject: [PATCH 1/4] Added has_question_event_happened_in_the_past --- .../agents/known_outcome_agent/deploy.py | 50 ++++++++++++------- .../known_outcome_agent.py | 42 +++++++++++++++- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index d364ed31..627ecb5d 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -1,6 +1,7 @@ import getpass from decimal import Decimal +from dotenv import load_dotenv from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.agent import DeployableAgent from prediction_market_agent_tooling.deploy.constants import OWNER_KEY @@ -17,6 +18,7 @@ from prediction_market_agent.agents.known_outcome_agent.known_outcome_agent import ( Result, get_known_outcome, + has_question_event_happened_in_the_past, ) @@ -25,7 +27,7 @@ def market_is_saturated(market: AgentMarket) -> bool: class DeployableKnownOutcomeAgent(DeployableAgent): - model = "gpt-4-1106-preview" + model = "gpt-3.5-turbo" def load(self) -> None: self.markets_with_known_outcomes: dict[str, Result] = {} @@ -37,6 +39,13 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: # been correctly bet on, and therefore the value of betting on them # is low. if not market_is_saturated(market=market): + + if not has_question_event_happened_in_the_past(model=self.model, + question=market.question): + # event happened + continue + + answer = get_known_outcome( model=self.model, question=market.question, @@ -62,22 +71,29 @@ def calculate_bet_amount(self, answer: bool, market: AgentMarket) -> BetAmount: if __name__ == "__main__": agent = DeployableKnownOutcomeAgent() - agent.deploy_gcp( - repository=f"git+{get_current_git_url()}@{get_current_git_commit_sha()}", + load_dotenv() + agent.deploy_local( market_type=MarketType.OMEN, - labels={OWNER_KEY: getpass.getuser()}, - secrets={ - "TAVILY_API_KEY": "GNOSIS_AI_TAVILY_API_KEY:latest", - }, - memory=1024, - api_keys=APIKeys( - BET_FROM_ADDRESS=verify_address( - "0xb611A9f02B318339049264c7a66ac3401281cc3c" - ), - BET_FROM_PRIVATE_KEY=private_key_type("EVAN_OMEN_BETTER_0_PKEY:latest"), - OPENAI_API_KEY=SecretStr("EVAN_OPENAI_API_KEY:latest"), - MANIFOLD_API_KEY=None, - ), - cron_schedule="0 */12 * * *", + sleep_time=10, timeout=540, + place_bet=False ) + # agent.deploy_gcp( + # repository=f"git+{get_current_git_url()}@{get_current_git_commit_sha()}", + # market_type=MarketType.OMEN, + # labels={OWNER_KEY: getpass.getuser()}, + # secrets={ + # "TAVILY_API_KEY": "GNOSIS_AI_TAVILY_API_KEY:latest", + # }, + # memory=1024, + # api_keys=APIKeys( + # BET_FROM_ADDRESS=verify_address( + # "0xb611A9f02B318339049264c7a66ac3401281cc3c" + # ), + # BET_FROM_PRIVATE_KEY=private_key_type("EVAN_OMEN_BETTER_0_PKEY:latest"), + # OPENAI_API_KEY=SecretStr("EVAN_OPENAI_API_KEY:latest"), + # MANIFOLD_API_KEY=None, + # ), + # cron_schedule="0 */12 * * *", + # timeout=540, + # ) diff --git a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py index 983d5bf9..492980b2 100644 --- a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py +++ b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py @@ -1,14 +1,16 @@ import json import typing as t -from datetime import datetime +from datetime import datetime, timezone from enum import Enum from langchain.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI from pydantic import BaseModel + from prediction_market_agent.tools.web_scrape.basic_summary import _summary from prediction_market_agent.tools.web_scrape.markdown import web_scrape +from prediction_market_agent_tooling.tools.utils import utcnow from prediction_market_agent.tools.web_search.tavily import web_search @@ -42,6 +44,22 @@ def has_known_outcome(self) -> bool: return self.result is not Result.UNKNOWN +HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT = """ +The current date is {date_str}. Your goal is to assert if a QUESTION references an event that is already finished (according to the current date and time) or if it will still take place in a later date. + +For example, you should return 1 if given the event "Will Bitcoin have reached the price of $100 by 30 March 2023?", since the event ends on a data prior to the current date. + +Your answer MUST be an integer and follow the logic below: +- If the event is already finished, return 1 +- If the event has not yet finished, return 0 +- If you are not sure, return -1 + +Answer with the single prompt only, and nothing else. + +[QUESTION] +"{question}" +""" + GENERATE_SEARCH_QUERY_PROMPT = """ The current date is {date_str}. You are trying to determine whether the answer to the following question has a definite answer. Generate a web search query @@ -137,6 +155,28 @@ def summarize_if_required(content: str, model: str, question: str) -> str: else: return content +def has_question_event_happened_in_the_past(model: str, question: str) -> bool: + """ Asks the model if the event referenced by the question has finished (given the + current date) (returning 1), if the event has not yet finished (returning 0) or + if it cannot be sure (returning -1).""" + date_str = utcnow().strftime("%Y-%m-%d %H:%M:%S %Z") + llm = ChatOpenAI(model=model, temperature=0.0) + prompt = ChatPromptTemplate.from_template( + template=HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT + ).format_messages( + date_str=date_str, + question=question, + ) + answer = str(llm.invoke(prompt).content) + try: + parsed_answer = int(answer) + if parsed_answer == 1: + return True + except Exception as e: + print ("Exception occured, cannot assert if title happened in the past. ", e) + + return False + def get_known_outcome(model: str, question: str, max_tries: int) -> Answer: """ From 475ecf6d2eb2b2de171ca9f7862edafbb12253e7 Mon Sep 17 00:00:00 2001 From: Gabriel Fior Date: Tue, 26 Mar 2024 18:00:38 -0300 Subject: [PATCH 2/4] Fixed CI steps --- .../agents/known_outcome_agent/deploy.py | 21 ++++-------------- .../known_outcome_agent.py | 22 +++++++++---------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index 627ecb5d..dd0a4571 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -1,19 +1,10 @@ -import getpass from decimal import Decimal from dotenv import load_dotenv -from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.agent import DeployableAgent -from prediction_market_agent_tooling.deploy.constants import OWNER_KEY -from prediction_market_agent_tooling.gtypes import SecretStr, private_key_type from prediction_market_agent_tooling.markets.agent_market import AgentMarket from prediction_market_agent_tooling.markets.data_models import BetAmount, Currency from prediction_market_agent_tooling.markets.markets import MarketType -from prediction_market_agent_tooling.tools.utils import ( - get_current_git_commit_sha, - get_current_git_url, -) -from prediction_market_agent_tooling.tools.web3_utils import verify_address from prediction_market_agent.agents.known_outcome_agent.known_outcome_agent import ( Result, @@ -39,12 +30,11 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: # been correctly bet on, and therefore the value of betting on them # is low. if not market_is_saturated(market=market): - - if not has_question_event_happened_in_the_past(model=self.model, - question=market.question): + if not has_question_event_happened_in_the_past( + model=self.model, question=market.question + ): # event happened continue - answer = get_known_outcome( model=self.model, @@ -73,10 +63,7 @@ def calculate_bet_amount(self, answer: bool, market: AgentMarket) -> BetAmount: agent = DeployableKnownOutcomeAgent() load_dotenv() agent.deploy_local( - market_type=MarketType.OMEN, - sleep_time=10, - timeout=540, - place_bet=False + market_type=MarketType.OMEN, sleep_time=10, timeout=540, place_bet=False ) # agent.deploy_gcp( # repository=f"git+{get_current_git_url()}@{get_current_git_commit_sha()}", diff --git a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py index 492980b2..f8a413ca 100644 --- a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py +++ b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py @@ -1,16 +1,15 @@ import json import typing as t -from datetime import datetime, timezone +from datetime import datetime from enum import Enum from langchain.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI +from prediction_market_agent_tooling.tools.utils import utcnow from pydantic import BaseModel - from prediction_market_agent.tools.web_scrape.basic_summary import _summary from prediction_market_agent.tools.web_scrape.markdown import web_scrape -from prediction_market_agent_tooling.tools.utils import utcnow from prediction_market_agent.tools.web_search.tavily import web_search @@ -155,26 +154,27 @@ def summarize_if_required(content: str, model: str, question: str) -> str: else: return content + def has_question_event_happened_in_the_past(model: str, question: str) -> bool: - """ Asks the model if the event referenced by the question has finished (given the + """Asks the model if the event referenced by the question has finished (given the current date) (returning 1), if the event has not yet finished (returning 0) or if it cannot be sure (returning -1).""" date_str = utcnow().strftime("%Y-%m-%d %H:%M:%S %Z") llm = ChatOpenAI(model=model, temperature=0.0) prompt = ChatPromptTemplate.from_template( - template=HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT - ).format_messages( - date_str=date_str, - question=question, - ) + template=HAS_QUESTION_HAPPENED_IN_THE_PAST_PROMPT + ).format_messages( + date_str=date_str, + question=question, + ) answer = str(llm.invoke(prompt).content) try: parsed_answer = int(answer) if parsed_answer == 1: return True except Exception as e: - print ("Exception occured, cannot assert if title happened in the past. ", e) - + print("Exception occured, cannot assert if title happened in the past. ", e) + return False From ba2105cbd1811f8364705a2da96bc9ec25481b42 Mon Sep 17 00:00:00 2001 From: Gabriel Fior Date: Tue, 26 Mar 2024 18:10:41 -0300 Subject: [PATCH 3/4] Reverted model back to gpt4 --- .../agents/known_outcome_agent/deploy.py | 52 ++++++++++--------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index dd0a4571..3a99483d 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -1,10 +1,18 @@ +import getpass from decimal import Decimal -from dotenv import load_dotenv +from prediction_market_agent_tooling.config import APIKeys from prediction_market_agent_tooling.deploy.agent import DeployableAgent +from prediction_market_agent_tooling.deploy.constants import OWNER_KEY +from prediction_market_agent_tooling.gtypes import SecretStr, private_key_type from prediction_market_agent_tooling.markets.agent_market import AgentMarket from prediction_market_agent_tooling.markets.data_models import BetAmount, Currency from prediction_market_agent_tooling.markets.markets import MarketType +from prediction_market_agent_tooling.tools.utils import ( + get_current_git_commit_sha, + get_current_git_url, +) +from prediction_market_agent_tooling.tools.web3_utils import verify_address from prediction_market_agent.agents.known_outcome_agent.known_outcome_agent import ( Result, @@ -18,7 +26,7 @@ def market_is_saturated(market: AgentMarket) -> bool: class DeployableKnownOutcomeAgent(DeployableAgent): - model = "gpt-3.5-turbo" + model = "gpt-4-1106-preview" def load(self) -> None: self.markets_with_known_outcomes: dict[str, Result] = {} @@ -61,26 +69,22 @@ def calculate_bet_amount(self, answer: bool, market: AgentMarket) -> BetAmount: if __name__ == "__main__": agent = DeployableKnownOutcomeAgent() - load_dotenv() - agent.deploy_local( - market_type=MarketType.OMEN, sleep_time=10, timeout=540, place_bet=False + agent.deploy_gcp( + repository=f"git+{get_current_git_url()}@{get_current_git_commit_sha()}", + market_type=MarketType.OMEN, + labels={OWNER_KEY: getpass.getuser()}, + secrets={ + "TAVILY_API_KEY": "GNOSIS_AI_TAVILY_API_KEY:latest", + }, + memory=1024, + api_keys=APIKeys( + BET_FROM_ADDRESS=verify_address( + "0xb611A9f02B318339049264c7a66ac3401281cc3c" + ), + BET_FROM_PRIVATE_KEY=private_key_type("EVAN_OMEN_BETTER_0_PKEY:latest"), + OPENAI_API_KEY=SecretStr("EVAN_OPENAI_API_KEY:latest"), + MANIFOLD_API_KEY=None, + ), + cron_schedule="0 */12 * * *", + timeout=540, ) - # agent.deploy_gcp( - # repository=f"git+{get_current_git_url()}@{get_current_git_commit_sha()}", - # market_type=MarketType.OMEN, - # labels={OWNER_KEY: getpass.getuser()}, - # secrets={ - # "TAVILY_API_KEY": "GNOSIS_AI_TAVILY_API_KEY:latest", - # }, - # memory=1024, - # api_keys=APIKeys( - # BET_FROM_ADDRESS=verify_address( - # "0xb611A9f02B318339049264c7a66ac3401281cc3c" - # ), - # BET_FROM_PRIVATE_KEY=private_key_type("EVAN_OMEN_BETTER_0_PKEY:latest"), - # OPENAI_API_KEY=SecretStr("EVAN_OPENAI_API_KEY:latest"), - # MANIFOLD_API_KEY=None, - # ), - # cron_schedule="0 */12 * * *", - # timeout=540, - # ) From dcf0de490b84cb24d0811be732fac4218039c945 Mon Sep 17 00:00:00 2001 From: Gabriel Fior Date: Wed, 27 Mar 2024 10:28:16 -0300 Subject: [PATCH 4/4] Added PR comments --- .../agents/known_outcome_agent/deploy.py | 12 +++++------- .../known_outcome_agent/known_outcome_agent.py | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/prediction_market_agent/agents/known_outcome_agent/deploy.py b/prediction_market_agent/agents/known_outcome_agent/deploy.py index 3a99483d..bbc0992e 100644 --- a/prediction_market_agent/agents/known_outcome_agent/deploy.py +++ b/prediction_market_agent/agents/known_outcome_agent/deploy.py @@ -37,13 +37,11 @@ def pick_markets(self, markets: list[AgentMarket]) -> list[AgentMarket]: # Assume very high probability markets are already known, and have # been correctly bet on, and therefore the value of betting on them # is low. - if not market_is_saturated(market=market): - if not has_question_event_happened_in_the_past( - model=self.model, question=market.question - ): - # event happened - continue - + if not market_is_saturated( + market=market + ) and has_question_event_happened_in_the_past( + model=self.model, question=market.question + ): answer = get_known_outcome( model=self.model, question=market.question, diff --git a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py index f8a413ca..e933e31b 100644 --- a/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py +++ b/prediction_market_agent/agents/known_outcome_agent/known_outcome_agent.py @@ -53,7 +53,7 @@ def has_known_outcome(self) -> bool: - If the event has not yet finished, return 0 - If you are not sure, return -1 -Answer with the single prompt only, and nothing else. +Answer with the single 1, 0 or -1 only, and nothing else. [QUESTION] "{question}"