Reduce rate throttle on v3 fetches. Clean up some stuff.

locast/candle_fetcher/dydx/api_fetcher/dydx_v3_fetcher.py

@@ -18,7 +18,7 @@
 
 class DydxV3Fetcher(APIFetcher):
     # TODO: Find smallest possible throttle
-    def __init__(self, client: Client, rate_throttle_sec: float = 0.8) -> None:
+    def __init__(self, client: Client, rate_throttle_sec: float = 0.4) -> None:
         self._exchange = Exchange.DYDX
         self._client = client
         self._mapper = ExchangeCandleMapper(DydxV3CandleMapping())

locast/candle_fetcher/dydx/candle_fetcher/dydx_v3_candle_fetcher.py

@@ -33,19 +33,19 @@ async def fetch_candles(
         end_date: datetime,
     ) -> List[Candle]:
         """
-        Fetches candles from the exchange (derived from api_fetcher) and market within a given time range.
+        Fetches candles from the derived exchange (api_fetcher) for specified market within the given time range.
 
         Args:
             market (str): The market to fetch candles for.
             resolution (str): The resolution of the candles to fetch.
             start_date (datetime): The start date of the time range (the started_at value of the oldest candle in the range).
-            end_date (datetime): The end date of the time range (the theoretical end date of the newest candle in the range).
+            end_date (datetime): The end date of the time range (the theoretical close date of the newest candle in the range).
 
         Returns:
             List[Candle]: A list of candles fetched from the specified exchange and market.
 
         Note: This function does not guarantee its returned candles to be reaching to the most recent existing candle on the exchange.
-        It simply fetches candles making up the range between the provided start_date and end_date. If this takes longer than newer
+        It simply fetches candles making up the range between the provided start_date and end_date at the time of fetching. If this takes longer than newer
         candles to be created on the exchange, those candles will not be fetched.
         """
         candles: List[Candle] = []
@@ -73,7 +73,7 @@ async def fetch_candles(
                     log_progress("🚛", "candles", "fetched", done, total)
 
         except Exception as e:
-            raise APIException(market, resolution, e) from e
+            raise APIException(self._exchange, market, resolution, e) from e
 
         return candles
 
@@ -101,8 +101,8 @@ async def fetch_candles_up_to_now(
         temp_start_date = start_date
         temp_norm_now = cu.normalized_now(resolution)
 
-        # This is what we wait for: The newest candle (at index 0) to have started_at one resolution below NOW,
-        # which only happens, if during fetch_candles a new candle started.
+        # This is what we wait for: The newest candle (at index 0) to have started_at exactly one resolution below NOW,
+        # which means, it is currently the newest finished candle. If below that value, it is already out of date.
         temp_now_minus_res = cu.subtract_n_resolutions(temp_norm_now, resolution, 1)
         while (not candles) or candles[0].started_at < temp_now_minus_res:
             new_candles = await self.fetch_candles(
@@ -143,11 +143,12 @@ def _create_log_vars(
 class APIException(Exception):
     def __init__(
         self,
+        exchange: Exchange,
         market: str,
         resolution: ResolutionDetail,
         exception: Exception,
         message: str | None = None,
     ) -> None:
         if message is None:
-            message = f"Error fetching market data for market '{market}' and resolution '{resolution.notation}: {exception}'."
+            message = f"Error ({exchange.name}) fetching market data for market '{market}' and resolution '{resolution.notation}: {exception}'."
         super().__init__(message)

notebooks/store_manger_example.ipynb

@@ -5,23 +5,15 @@
    "metadata": {},
    "source": [
     "# Intro\n",
-    "Running this notebook will create the file `locast.db` in the notebooks folder, containing a price candle cluster for ETH-USD. Once it exists, re-running this notebook will attempt to update that cluster. Depending on how much time has passed, this will not be necessary yet. "
+    "Running this notebook will create the file `locast.db` in the notebooks folder, containing a price candle cluster for ETH-USD. \n",
+    "Once it exists, re-running this notebook will attempt to update that cluster. Depending on how much time has passed, this might not be necessary yet. "
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "♻️ Cluster is already up to date. Try again after 2024-09-26 14:15:00+00:00.\n",
-      "🕯️ETH-USD-Candles in cluster: 10002.\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datetime import timedelta\n",
     "import time\n",
@@ -64,7 +56,7 @@
     "    # Define desired candle cluster parameters.\n",
     "    exchange = Exchange.DYDX_V4\n",
     "    resolution = DydxResolution.ONE_MINUTE\n",
-    "    market = \"ETH-USD\"\n",
+    "    market = \"BTC-USD\"\n",
     "\n",
     "    # Define start date for candle cluster.\n",
     "    now = cu.normalized_now(resolution)  # Now, rounded down to nearest resolution tick\n",
@@ -128,7 +120,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.4"
+   "version": "3.11.2"
   }
  },
  "nbformat": 4,

tests/conftest.py

@@ -67,7 +67,7 @@ def dydx_v4_eth_one_min_mock_candles() -> Generator[list[Candle], None, None]:
     mapper = ExchangeCandleMapper(DydxV4CandleMapping())
     yield mapper.to_candles(eth_dicts)
 
-
+# region - dYdX v4
 @pytest_asyncio.fixture  # type: ignore
 async def dydx_v4_fetcher_mock() -> AsyncGenerator[DydxV4Fetcher, None]:
     mock_client = V4IndexerClientMock()
@@ -93,7 +93,7 @@ async def dydx_v4_candle_fetcher_mainnet() -> AsyncGenerator[DydxV4CandleFetcher
     yield DydxV4CandleFetcher(api_fetcher=DydxV4Fetcher(mainnet_client))
 
 
-# dydx v3
+# region - dYdX v3
 @pytest_asyncio.fixture  # type: ignore
 async def dydx_v3_fetcher_mock() -> AsyncGenerator[DydxV3Fetcher, None]:
     mock_client = V3ClientMock()