From 78dc856e77c8f3637179b22c7b9af036a2cb16be Mon Sep 17 00:00:00 2001 From: ericLemanissier Date: Mon, 22 May 2023 13:09:54 +0200 Subject: [PATCH] handle Retry-After header --- ccb/github.py | 15 +++++++++++++++ ccb/upstream_project.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ccb/github.py b/ccb/github.py index 5c99e8c6..3abc1324 100644 --- a/ccb/github.py +++ b/ccb/github.py @@ -1,3 +1,8 @@ +import logging +import requests + +logger = logging.getLogger(__name__) + class _GitHubToken: value = None @@ -8,3 +13,13 @@ def get_github_token(): def set_github_token(token): _GitHubToken.value = token + print_github_token_rate_limit() + + +def print_github_token_rate_limit(): + github_token = get_github_token() + headers = {"Accept": "application/vnd.github.v3+json"} + if github_token: + headers["Authorization"] = f"token {github_token}" + res = requests.get(" https://api.github.com/rate_limit", headers=headers, timeout=1) + logger.warning("github rate limits: %s", res.json()) diff --git a/ccb/upstream_project.py b/ccb/upstream_project.py index c0e347dd..4aab6ef9 100644 --- a/ccb/upstream_project.py +++ b/ccb/upstream_project.py @@ -19,7 +19,7 @@ ) from .subprocess import check_output, check_call from .utils import SemaphoneStorage, format_duration -from .github import get_github_token +from .github import get_github_token, print_github_token_rate_limit logger = logging.getLogger(__name__) @@ -263,11 +263,18 @@ def _get_url_for_version(v): continue r.url = _get_url_for_version(v) self.__versions.append(r) - + + except aiohttp.ClientResponseError as exc: + for h in exc.headers: + if h == "Retry-After": + time.sleep(int(exc.headers[h])) + return await self.versions() + logger.info("%s: error parsing repository: %s", self.recipe.name, exc) + self.__versions = [] except Exception as exc: logger.info("%s: error parsing repository: %s", self.recipe.name, exc) logger.debug(traceback.format_exc()) - self.__versions = [] + self.__versions = [] if self.__versions: return self.__versions else: @@ -340,6 +347,11 @@ def _get_url_for_version(v): continue r.url = _get_url_for_version(v) self.__versions.append(r) + except aiohttp.ClientResponseError as exc: + logger.info("%s: error parsing repository: %s", self.recipe.name, exc) + if exc.status == 429: + logger.warning("gitlab rate limited %s", exc.headers) + self.__versions = [] except Exception as exc: logger.info("%s: error parsing repository: %s", self.recipe.name, exc) logger.debug(traceback.format_exc())