From 11559270fe7db4c7e59f70399267369bf1ba5687 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0pl=C3=ADchal?= Date: Fri, 4 Oct 2024 15:37:46 +0200 Subject: [PATCH] Handle the GitHub rate limit in a better way (#374) Let's use the `X-RateLimit-Reset` header provided by GitHub and sleep until the suggested time to complete the report instead of bailing out in the middle. --- did/plugins/github.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/did/plugins/github.py b/did/plugins/github.py index 31621471..b2c74eaa 100644 --- a/did/plugins/github.py +++ b/did/plugins/github.py @@ -24,6 +24,7 @@ import json import re +import time import requests @@ -77,16 +78,21 @@ def search(self, query): "Defined token is not valid. " "Either update it or remove it.") + # Handle the exceeded rate limit + if response.status_code in [403, 429]: + if response.headers.get("X-RateLimit-Remaining") == "0": + reset_time = int(response.headers["X-RateLimit-Reset"]) + sleep_time = int(max(reset_time - time.time(), 0)) + 1 + log.warning("GitHub rate limit exceeded, use token to speed up.") + log.warning(f"Sleeping now for {listed(sleep_time, 'second')}.") + time.sleep(sleep_time) + continue + raise ReportError(f"GitHub query failed: {response.text}") + # Parse fetched json data try: data = json.loads(response.text)["items"] result.extend(data) - except KeyError: - if json.loads(response.text)["message"].startswith( - "API rate limit exceeded"): - raise ReportError( - "GitHub API rate limit exceeded. " - "Consider creating an access token.") except requests.exceptions.JSONDecodeError as error: log.debug(error) raise ReportError(f"GitHub JSON failed: {response.text}.")