diff --git a/src/ape/utils/__init__.py b/src/ape/utils/__init__.py index ed8cc1bd87..8959b4c500 100644 --- a/src/ape/utils/__init__.py +++ b/src/ape/utils/__init__.py @@ -84,7 +84,13 @@ def __getattr__(name: str): return getattr(process_module, name) - elif name in ("USER_AGENT", "RPCHeaders", "allow_disconnected", "stream_response"): + elif name in ( + "USER_AGENT", + "RPCHeaders", + "allow_disconnected", + "request_with_retry", + "stream_response", + ): import ape.utils.rpc as rpc_module return getattr(rpc_module, name) @@ -166,6 +172,7 @@ def __getattr__(name: str): "path_match", "raises_not_implemented", "returns_array", + "request_with_retry", "RPCHeaders", "run_in_tempdir", "run_until_complete", diff --git a/src/ape/utils/rpc.py b/src/ape/utils/rpc.py index f552fb1c86..a98b706fc6 100644 --- a/src/ape/utils/rpc.py +++ b/src/ape/utils/rpc.py @@ -1,10 +1,13 @@ +import time from collections.abc import Callable +from random import randint +from typing import Optional import requests from requests.models import CaseInsensitiveDict from tqdm import tqdm # type: ignore -from ape.exceptions import ProviderNotConnectedError +from ape.exceptions import ProviderError, ProviderNotConnectedError from ape.logging import logger from ape.utils.misc import __version__, _python_version @@ -89,3 +92,60 @@ def __setitem__(self, key, value): if new_user_agent := " ".join(new_parts): super().__setitem__(key, f"{existing_user_agent} {new_user_agent}") + + +def request_with_retry( + func: Callable, + min_retry_delay: int = 1_000, + retry_backoff_factor: int = 2, + max_retry_delay: int = 30_000, + max_retries: int = 10, + retry_jitter: int = 250, + is_rate_limit: Optional[Callable[[Exception], bool]] = None, +): + """ + Make a request with 429/rate-limit retry logic. + + Args: + func (Callable): The function to run with rate-limit handling logic. + min_retry_delay (int): The amount of milliseconds to wait before + retrying the request. Defaults to ``1_000`` (one second). + retry_backoff_factor (int): The multiplier applied to the retry delay + after each failed attempt. Defaults to ``2``. + max_retry_delay (int): The maximum length of the retry delay. + Defaults to ``30_000`` (30 seconds). + max_retries (int): The maximum number of retries. + Defaults to ``10``. + retry_jitter (int): A random number of milliseconds up to this limit + is added to each retry delay. Defaults to ``250`` milliseconds. + is_rate_limit (Callable[[Exception], bool] | None): A custom handler + for detecting rate-limits. Defaults to checking for a 429 status + code on an HTTPError. + """ + if not is_rate_limit: + # Use default checker. + def checker(err: Exception) -> bool: + return isinstance(err, requests.HTTPError) and err.response.status_code == 429 + + is_rate_limit = checker + + for attempt in range(max_retries): + try: + return func() + except Exception as err: + if not is_rate_limit(err): + # It was not a rate limit error. Raise whatever exception it is. + raise + + else: + # We were rate-limited. Invoke retry/backoff logic. + logger.warning("Request was rate-limited. Backing-off and then retrying...") + retry_interval = min( + max_retry_delay, min_retry_delay * retry_backoff_factor**attempt + ) + delay = retry_interval + randint(0, retry_jitter) + time.sleep(delay / 1000) + continue + + # If we get here, we over-waited. Raise custom exception. + raise ProviderError(f"Rate limit retry-mechanism exceeded after '{max_retries}' attempts.") diff --git a/src/ape_ethereum/provider.py b/src/ape_ethereum/provider.py index 7fbb254f2d..bd14077935 100644 --- a/src/ape_ethereum/provider.py +++ b/src/ape_ethereum/provider.py @@ -30,7 +30,7 @@ ) try: - from web3.exceptions import Web3RPCError + from web3.exceptions import Web3RPCError # type: ignore except ImportError: Web3RPCError = ValueError # type: ignore @@ -66,6 +66,7 @@ from ape.utils._web3_compat import ExtraDataToPOAMiddleware, WebsocketProvider from ape.utils.basemodel import ManagerAccessMixin from ape.utils.misc import DEFAULT_MAX_RETRIES_TX, gas_estimation_error_message, to_int +from ape.utils.rpc import request_with_retry from ape_ethereum._print import CONSOLE_ADDRESS, console_contract from ape_ethereum.trace import CallTrace, TraceApproach, TransactionTrace from ape_ethereum.transactions import AccessList, AccessListTransaction, TransactionStatusEnum @@ -1134,8 +1135,10 @@ def _post_connect(self): self.chain_manager.contracts._cache_contract_type(CONSOLE_ADDRESS, console_contract) def make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any: - parameters = parameters or [] + return request_with_retry(lambda: self._make_request(rpc, parameters=parameters)) + def _make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any: + parameters = parameters or [] try: result = self.web3.provider.make_request(RPCEndpoint(rpc), parameters) except HTTPError as err: @@ -1144,6 +1147,9 @@ def make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any: f"RPC method '{rpc}' is not implemented by this node instance." ) + elif err.response.status_code == 429: + raise # Raise as-is so rate-limit handling picks it up. + raise ProviderError(str(err)) from err if "error" in result: diff --git a/tests/functional/test_provider.py b/tests/functional/test_provider.py index 6d0455e170..4f17df17d1 100644 --- a/tests/functional/test_provider.py +++ b/tests/functional/test_provider.py @@ -22,7 +22,12 @@ ) from ape.types.events import LogFilter from ape.utils.testing import DEFAULT_TEST_ACCOUNT_BALANCE, DEFAULT_TEST_CHAIN_ID -from ape_ethereum.provider import WEB3_PROVIDER_URI_ENV_VAR_NAME, Web3Provider, _sanitize_web3_url +from ape_ethereum.provider import ( + WEB3_PROVIDER_URI_ENV_VAR_NAME, + EthereumNodeProvider, + Web3Provider, + _sanitize_web3_url, +) from ape_ethereum.transactions import TransactionStatusEnum, TransactionType from ape_test import LocalProvider @@ -468,6 +473,33 @@ def custom_make_request(rpc, params): eth_tester_provider._web3 = real_web3 +def test_make_request_rate_limiting(mocker, ethereum, mock_web3): + provider = EthereumNodeProvider(network=ethereum.local) + provider._web3 = mock_web3 + + class RateLimitTester: + tries = 3 + _try = 0 + tries_made = 0 + + def rate_limit_hook(self, rpc, params): + self.tries_made += 1 + if self._try >= self.tries: + self._try = 0 + return {"success": True} + else: + self._try += 1 + response = mocker.MagicMock() + response.status_code = 429 + raise HTTPError(response=response) + + rate_limit_tester = RateLimitTester() + mock_web3.provider.make_request.side_effect = rate_limit_tester.rate_limit_hook + result = provider.make_request("ape_testRateLimiting", parameters=[]) + assert rate_limit_tester.tries_made == rate_limit_tester.tries + 1 + assert result == {"success": True} + + def test_base_fee(eth_tester_provider): actual = eth_tester_provider.base_fee assert actual > 0