Polygon day fix

lumibot/backtesting/backtesting_broker.py

@@ -3,9 +3,10 @@
 from datetime import timedelta
 from decimal import Decimal
 from functools import wraps
-
+from lumibot import LUMIBOT_DEFAULT_TIMEZONE
 import pandas as pd
 
+
 from lumibot.brokers import Broker
 from lumibot.data_sources import DataSourceBacktesting
 from lumibot.entities import Asset, Order, Position, TradingFee
@@ -111,24 +112,12 @@ def should_continue(self):
 
     def is_market_open(self):
         """Return True if market is open else false"""
-        now = self.datetime
-
-        # As the index is sorted, use searchsorted to find the relevant day
-        idx = self._trading_days.index.searchsorted(now, side='right')
-
-        # Check that the index is not out of bounds
-        if idx >= len(self._trading_days):
-            logging.error("Cannot predict future")
+        market_open, market_close = self._get_market_hours(self.datetime)
+
+        if market_open is None:
             return False
-
-        # The index of the trading_day is used as the market close time
-        market_close = self._trading_days.index[idx]
-
-        # Retrieve market open time using .at since idx is a valid datetime index
-        market_open = self._trading_days.at[market_close, 'market_open']
-
-        # Check if 'now' is within the trading hours of the located day
-        return market_open <= now < market_close
+
+        return market_open <= self.datetime < market_close
 
     def _get_next_trading_day(self):
         now = self.datetime
@@ -141,66 +130,97 @@ def _get_next_trading_day(self):
     def get_time_to_open(self):
         """Return the remaining time for the market to open in seconds"""
         now = self.datetime
+        today = pd.Timestamp(now.date())
 
-        search = self._trading_days[now < self._trading_days.index]
-        if search.empty:
+        idx = self._trading_days.index.searchsorted(today)
+
+        if idx >= len(self._trading_days):
             logging.error("Cannot predict future")
             return 0
 
-        trading_day = search.iloc[0]
-        open_time = trading_day.market_open
-
+        market_open = self._trading_days.market_open.iloc[idx]
+
+        #search = self._trading_days[now < self._trading_days.index]
+
         # For Backtesting, sometimes the user can just pass in dates (i.e. 2023-08-01) and not datetimes
         # In this case the "now" variable is starting at midnight, so we need to adjust the open_time to be actual
         # market open time.  In the case where the user passes in a time inside a valid trading day, use that time
         # as the start of trading instead of market open.
-        if self.IS_BACKTESTING_BROKER and now > open_time:
-            open_time = self.data_source.datetime_start
+        if self.IS_BACKTESTING_BROKER and now > market_open:
+            market_open = self.data_source.datetime_start
 
-        if now >= open_time:
+        if now >= market_open:
             return 0
 
-        delta = open_time - now
+        delta = market_open - now
         return delta.total_seconds()
 
     def get_time_to_close(self):
         """Return the remaining time for the market to close in seconds"""
         now = self.datetime
+        today = pd.Timestamp(now.date())
 
         # Use searchsorted for efficient searching and reduce unnecessary DataFrame access
-        idx = self._trading_days.index.searchsorted(now, side='left')
+        idx = self._trading_days.index.searchsorted(today)
 
         if idx >= len(self._trading_days):
             logging.error("Cannot predict future")
             return 0
 
         # Directly access the data needed using more efficient methods
-        market_close_time = self._trading_days.index[idx]
-        market_open = self._trading_days.at[market_close_time, 'market_open']
-        market_close = market_close_time  # Assuming this is a scalar value directly from the index
+        market_open = self._trading_days.market_open.iloc[idx]
+        market_close = self._trading_days.market_close.iloc[idx]
 
         if now < market_open:
             return None
 
         delta = market_close - now
         return delta.total_seconds()
 
-    def _await_market_to_open(self, timedelta=None, strategy=None):
-        if self.data_source.SOURCE == "PANDAS" and self.data_source._timestep == "day":
-            return
+    def _get_market_hours(self, dt):
+        today = pd.Timestamp(dt.date())
 
+        try:
+            idx = self._trading_days.index.get_loc(today)
+        except KeyError:
+            return [None, None]
+
+        idx = self._trading_days.index.get_loc(today)
+
+        market_open = self._trading_days.market_open.iloc[idx]    
+        market_close = self._trading_days.market_close.iloc[idx]
+
+        return [market_open, market_close]
+
+    def _await_market_to_open(self, minutes_delta=None, strategy=None):
         # Process outstanding orders first before waiting for market to open
         # or else they don't get processed until the next day
         self.process_pending_orders(strategy=strategy)
-
-        time_to_open = self.get_time_to_open()
-        if timedelta:
-            time_to_open -= 60 * timedelta
-        self._update_datetime(time_to_open)
+
+        is_market_closed = False
+        market_open, market_close = self._get_market_hours(self.datetime)
+
+        if market_open is None:
+            is_market_closed = True
+        elif self.datetime >= market_close:
+            is_market_closed = True
+
+        # If market closed get next day!
+        while is_market_closed:
+            today = pd.Timestamp(self.datetime.date())
+            tomorrow = (today + timedelta(days=1)).tz_localize(LUMIBOT_DEFAULT_TIMEZONE)
+            self._update_datetime(tomorrow)
+            market_open, market_close = self._get_market_hours(self.datetime)
+            if market_open:
+                is_market_closed = False
+
+        if minutes_delta:
+            market_open -= timedelta(seconds = 60 * minutes_delta)
+        self._update_datetime(market_open)
 
     def _await_market_to_close(self, timedelta=None, strategy=None):
-        if self.data_source.SOURCE == "PANDAS" and self.data_source._timestep == "day":
-            return
+        #if self.data_source.SOURCE == "PANDAS" and self.data_source._timestep == "day":
+        #    return
 
         # Process outstanding orders first before waiting for market to close
         # or else they don't get processed until the next day
@@ -611,7 +631,7 @@ def process_pending_orders(self, strategy):
                     2,
                     quote=order.quote,
                     timeshift=-2,
-                    timestep=self.data_source._timestep,
+                    timestep=self.data_source._timestep
                 )
                 # Check if we got any ohlc data
                 if ohlc is None:

lumibot/backtesting/polygon_backtesting.py

@@ -31,10 +31,14 @@ def __init__(
         pandas_data=None,
         api_key=None,
         has_paid_subscription=False,
+        timestep='minute',
         **kwargs,
     ):
+        # This should be better in Data class
+        timestep = self._parse_source_timestep(timestep)
+
         super().__init__(
-            datetime_start=datetime_start, datetime_end=datetime_end, pandas_data=pandas_data, api_key=api_key, **kwargs
+            datetime_start=datetime_start, datetime_end=datetime_end, timestep=timestep, pandas_data=pandas_data, api_key=api_key, **kwargs
         )
         self.has_paid_subscription = has_paid_subscription
 
@@ -209,6 +213,7 @@ def _pull_source_symbol_bars(
         exchange: str = None,
         include_after_hours: bool = True,
     ):
+
         # Get the current datetime and calculate the start datetime
         current_dt = self.get_datetime()
 
@@ -242,7 +247,8 @@ def get_historical_prices_between_dates(
         bars = self._parse_source_symbol_bars(response, asset, quote=quote)
         return bars
 
-    def get_last_price(self, asset, timestep="minute", quote=None, exchange=None, **kwargs):
+    def get_last_price(self, asset, timestep=None, quote=None, exchange=None, **kwargs):
+        timestep = timestep or self.get_timestep()
         try:
             dt = self.get_datetime()
             self._update_pandas_data(asset, quote, 1, timestep, dt)

lumibot/brokers/broker.py

@@ -13,6 +13,8 @@
 from dateutil import tz
 from termcolor import colored
 
+
+from lumibot import LUMIBOT_DEFAULT_TIMEZONE
 from lumibot.data_sources import DataSource
 from lumibot.entities import Asset, Order, Position
 from lumibot.trading_builtins import SafeList
@@ -691,6 +693,16 @@ def should_continue(self):
 
     def market_close_time(self):
         return self.utc_to_local(self.market_hours(close=True))
+
+    def is_market_day(self):
+        if self._trading_days is None:
+            return True
+
+        trading_days_dates = self._trading_days.index.normalize()
+
+        today = pd.Timestamp(self.datetime.date()).tz_localize(LUMIBOT_DEFAULT_TIMEZONE)
+
+        return today in trading_days_dates
 
     def is_market_open(self):
         """Determines if the market is open.

lumibot/data_sources/data_source_backtesting.py

@@ -32,9 +32,6 @@ def __init__(
         self._iter_count = None
         self.backtesting_started = _backtesting_started
 
-        # Subtract one minute from the datetime_end so that the strategy stops right before the datetime_end
-        self.datetime_end -= timedelta(minutes=1)
-
         # Legacy strategy.backtest code will always pass in a config even for DataSources that don't need it, so
         # catch it here and ignore it in this class. Child classes that need it should error check it themselves.
         self._config = config

lumibot/data_sources/pandas_data.py

@@ -19,15 +19,15 @@ class PandasData(DataSourceBacktesting):
         {"timestep": "minute", "representations": ["1M", "minute"]},
     ]
 
-    def __init__(self, *args, pandas_data=None, auto_adjust=True, **kwargs):
+    def __init__(self, *args, pandas_data=None, auto_adjust=True, timestep="minute", **kwargs):
         super().__init__(*args, **kwargs)
         self.name = "pandas"
         self.pandas_data = self._set_pandas_data_keys(pandas_data)
         self.auto_adjust = auto_adjust
         self._data_store = self.pandas_data
         self._date_index = None
         self._date_supply = None
-        self._timestep = "minute"
+        self._timestep = timestep
 
     @staticmethod
     def _set_pandas_data_keys(pandas_data):
@@ -158,9 +158,16 @@ def update_date_index(self):
                 dt_index = dt_index.join(data.df.index, how="outer")
 
         if dt_index is None:
-            # Build a dummy index
-            freq = "1min" if self._timestep == "minute" else "1D"
-            dt_index = pd.date_range(start=self.datetime_start, end=self.datetime_end, freq=freq)
+            # Determine the frequency and adjust the end date accordingly
+            if self._timestep == "minute":
+                freq = "1min"
+                adjusted_end = self.datetime_end
+            else:  # Assumes the only other option is daily
+                freq = "1D"
+                adjusted_end = self.datetime_end + pd.Timedelta(days=1)
+
+            # Generate date range with the determined frequency and adjusted end date
+            dt_index = pd.date_range(start=self.datetime_start, end=adjusted_end, freq=freq)
 
         else:
             if self.datetime_end < dt_index[0]:

lumibot/entities/data.py

@@ -222,10 +222,12 @@ def columns(self, df):
     def set_date_format(self, df):
         df.index.name = "datetime"
         df.index = pd.to_datetime(df.index)
+
         if not df.index.tzinfo:
             df.index = df.index.tz_localize(DEFAULT_PYTZ)
         elif df.index.tzinfo != DEFAULT_PYTZ:
             df.index = df.index.tz_convert(DEFAULT_PYTZ)
+
         return df
 
     def set_dates(self, date_start, date_end):
@@ -517,6 +519,10 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0):
             unit = "D"
             data = self._get_bars_dict(dt, length=length, timestep="minute", timeshift=timeshift)
 
+        elif timestep == "day" and self.timestep == "day":
+            length = length
+            unit = "D"
+            data = self._get_bars_dict(dt, length=length, timestep="day", timeshift=timeshift)
         else:
             unit = "min"  # Guaranteed to be minute timestep at this point
             length = length * quantity
@@ -532,7 +538,7 @@ def get_bars(self, dt, length=1, timestep=MIN_TIMESTEP, timeshift=0):
         df_result = df_result.dropna()
 
         # Remove partial day data from the current day, which can happen if the data is in minute timestep.
-        if timestep == "day":
+        if timestep == "day" and self.timestep == "minute":
             df_result = df_result[df_result.index < dt.replace(hour=0, minute=0, second=0, microsecond=0)]
 
         # The original df_result may include more rows when timestep is day and self.timestep is minute.

lumibot/strategies/strategy.py

@@ -921,6 +921,17 @@ def set_market(self, market):
 
         self.broker.market = market
 
+    def is_market_day(self):
+        """
+        Determine if the current day is a market day.
+
+        Returns
+        -------
+        bool
+            True if today is a market day, otherwise False.
+        """
+        return self.broker.is_market_day()
+
     def await_market_to_open(self, timedelta=None):
         """Executes infinite loop until market opens
 
@@ -2988,7 +2999,7 @@ def get_historical_prices(
         self,
         asset: Union[Asset, str],
         length: int,
-        timestep: str = "",
+        timestep: str = None,
         timeshift: datetime.timedelta = None,
         quote: Asset = None,
         exchange: str = None,
@@ -3080,13 +3091,12 @@ def get_historical_prices(
         if quote is None:
             quote = self.quote_asset
 
-        self.logger.info(f"Getting historical prices for {asset}, {length} bars, {timestep}")
+        self.logger.info(f"Getting historical prices for {asset}, {length} bars, {timestep if timestep is not None else ''}")
 
         asset = self._sanitize_user_asset(asset)
 
         asset = self.crypto_assets_to_tuple(asset, quote)
-        if not timestep:
-            timestep = self.broker.data_source.MIN_TIMESTEP
+
         return self.broker.data_source.get_historical_prices(
             asset,
             length,