Fix for pandas pre 2.2.0

Fixes to accommodate pandas pre 2.2.0 default frequency strings, "T"
 (now "min"), "H" (now "h"), "S" (now "s"), "M" (now "ME") and "Y"
(now "YE").

Author: maread99

.github/workflows/release.yml

@@ -37,6 +37,7 @@ jobs:
     - name: Install from testpypi and import
       shell: bash
       run: |
+        sleep 5
         i=0
         while [ $i -lt 12 ] && [ "${{ github.ref_name }}" != $(pip index versions -i https://test.pypi.org/simple --pre market_prices | cut -d'(' -f2 | cut -d')' -f1 | sed 1q) ];\
           do echo "waiting for package to appear in test index, i is $i"; echo "sleeping 5s"; sleep 5s; echo "woken up"; let i++; echo "next i is $i"; done

src/market_prices/intervals.py

@@ -56,7 +56,13 @@ def freq_unit(self) -> typing.Literal["min", "h", "D"]:
 
         Returns either "min", "h" or "D".
         """
-        return self.as_pdtd.resolution_string
+        unit = self.as_pdtd.resolution_string
+        # for pre pandas 2.2 compatibility...
+        if unit == "T":
+            unit = "min"
+        if unit == "H":
+            unit = "h"
+        return unit
 
     @property
     def freq_value(self) -> int:
@@ -449,8 +455,7 @@ def to_ptinterval(interval: str | timedelta | pd.Timedelta) -> PTInterval:
             " interval in terms of months pass as a string, for"
             ' example "1m" for one month.'
         )
-
-        valid_resolutions = ["min", "h", "D"]
+        valid_resolutions = ["min", "h", "D"] + ["T", "H"]  # + form pandas pre 2.2
         if interval.resolution_string not in valid_resolutions:
             raise ValueError(error_msg)
 

src/market_prices/pt.py

@@ -1480,7 +1480,9 @@ def downsample(  # pylint: disable=arguments-differ
             else:
                 return self._downsample_days(pdfreq)
 
-        if unit in ["h", "min", "s", "L", "ms", "U", "us", "N", "ns"]:
+        invalid_units = ["h", "min", "MIN", "s", "L", "ms", "U", "us", "N", "ns"]
+        ext = ["t", "T", "H", "S"]  # for pandas pre 2.2 compatibility
+        if unit in invalid_units + ext:
             raise ValueError(
                 "Cannot downsample to a `pdfreq` with a unit more precise than 'd'."
             )
@@ -2328,6 +2330,12 @@ def downsample(
             )
 
         unit = genutils.remove_digits(pdfreq)
+        # for pandas pre 2.2. compatibility
+        if unit == "T":
+            unit = "min"
+        if unit == "H":
+            unit = "h"
+
         valid_units = ["min", "h"]
         if unit not in valid_units:
             raise ValueError(

src/market_prices/utils/pandas_utils.py

@@ -76,6 +76,19 @@ def timestamps_in_interval_of_intervals(
 
     Examples
     --------
+    >>> # ignore first part, for testing purposes only...
+    >>> import pytest, pandas
+    >>> v = pandas.__version__
+    >>> if (
+    ...     (v.count(".") == 1 and float(v) < 2.2)
+    ...     or (
+    ...         v.count(".") > 1
+    ...         and float(v[:v.index(".", v.index(".") + 1)]) < 2.2
+    ...     )
+    ... ):
+    ...     pytest.skip("printed return only valid from pandas 2.2")
+    >>> #
+    >>> # example from here...
     >>> timestamps = pd.DatetimeIndex(
     ...     [
     ...         pd.Timestamp('2021-03-12 14:00'),
@@ -96,6 +109,7 @@ def timestamps_in_interval_of_intervals(
     >>> timestamps_in_interval_of_intervals(timestamps, intervals)
     True
     """
+    # NOTE Can lose doctest skip when pandas support is >= 2.2
     timestamps = [timestamps] if isinstance(timestamps, pd.Timestamp) else timestamps
     ser = intervals.to_series()
     bv = ser.apply(lambda x: all({ts in x for ts in timestamps}))
@@ -387,6 +401,19 @@ def remove_intervals_from_interval(
 
     Examples
     --------
+    >>> # ignore first part, for testing purposes only...
+    >>> import pytest, pandas
+    >>> v = pandas.__version__
+    >>> if (
+    ...     (v.count(".") == 1 and float(v) < 2.2)
+    ...     or (
+    ...         v.count(".") > 1
+    ...         and float(v[:v.index(".", v.index(".") + 1)]) < 2.2
+    ...     )
+    ... ):
+    ...     pytest.skip("printed return only valid from pandas 2.2")
+    >>> #
+    >>> # example from here...
     >>> from pprint import pprint
     >>> left = pd.date_range('2021-05-01 12:00', periods=5, freq='h')
     >>> right = left + pd.Timedelta(30, 'min')
@@ -411,6 +438,7 @@ def remove_intervals_from_interval(
      Interval(2021-05-01 15:30:00, 2021-05-01 16:00:00, closed='left'),
      Interval(2021-05-01 16:30:00, 2021-05-01 17:30:00, closed='left')]
     """
+    # NOTE Can lose doctest skip when pandas support is >= 2.2
     if not intervals.is_monotonic_increasing:
         raise ValueError(
             "`intervals` must be monotonically increasing although receieved"

tests/conftest.py

@@ -280,3 +280,16 @@ def xlon_calendar_extended(
 def xhkg_calendar(today, side, mock_now) -> abc.Iterator[xcals.ExchangeCalendar]:
     """XLON calendar."""
     yield xcals.get_calendar("XHKG", side=side, end=today)
+
+
+@pytest.fixture
+def pandas_pre_22() -> abc.Iterator[bool]:
+    """Installed pandas is pre version 2.2."""
+    v = pd.__version__
+    if v.count(".") == 1:
+        rtrn = float(v) < 2.2
+    else:
+        stop = v.index(".", v.index(".") + 1)
+        minor_v = float(v[:stop])
+        rtrn = minor_v < 2.2
+    yield rtrn

tests/hypstrtgy.py

@@ -361,6 +361,16 @@ def pp_days(draw, calendar_name: str) -> st.SearchStrategy[dict[str, typing.Any]
     return pp
 
 
+# set PRE_PANDAS_22
+v = pd.__version__
+if v.count(".") == 1:
+    PRE_PANDAS_22 = float(v) < 2.2
+else:
+    stop = v.index(".", v.index(".") + 1)
+    minor_v = float(v[:stop])
+    PRE_PANDAS_22 = minor_v < 2.2
+
+
 @st.composite
 def pp_days_start_session(
     draw,
@@ -387,7 +397,10 @@ def pp_days_start_session(
     sessions = calendar.sessions
     limit_r = sessions[-pp["days"]]
     if start_will_roll_to_ms:
-        offset = pd.tseries.frequencies.to_offset("ME")
+        # NOTE when min pandas support moves to >= 2.2 can hard code this as ME
+        # and lose the PRE_PANDAS_22 global.
+        freq = "M" if PRE_PANDAS_22 else "ME"
+        offset = pd.tseries.frequencies.to_offset(freq)
         if TYPE_CHECKING:
             assert offset is not None
         limit_r = offset.rollback(limit_r)

tests/test_pt.py

@@ -2289,7 +2289,7 @@ def xnys_open(self, xnys, session) -> abc.Iterator[pd.Timestamp]:
     def xnys_close(self, xnys, session) -> abc.Iterator[pd.Timestamp]:
         yield xnys.session_close(session)
 
-    def test_errors(self, intraday_pt, composite_intraday_pt, one_min):
+    def test_errors(self, intraday_pt, composite_intraday_pt, one_min, pandas_pre_22):
         """Verify raising expected errors for intraday price table."""
         df = intraday_pt
         f = df.pt.downsample
@@ -2328,7 +2328,9 @@ def match_f(pdfreq) -> str:
                 f" received `pdfreq` as {pdfreq}."
             )
 
-        invalid_pdfreqs = ["1d", "1s", "1ns", "1ms", "1ME", "1YE"]
+        invalid_pdfreqs = ["1d", "1s", "1ns", "1ms"]
+        ext = ["1M", "1Y"] if pandas_pre_22 else ["1ME", "1YE"]
+        invalid_pdfreqs += ext
         for pdfreq in invalid_pdfreqs:
             with pytest.raises(ValueError, match=match_f(pdfreq)):
                 f(pdfreq)