add user-agent

also, more debug in case of rate limiting

ccb/github.py

@@ -1,3 +1,8 @@
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+
 class _GitHubToken:
     value = None
 
@@ -8,3 +13,13 @@ def get_github_token():
 
 def set_github_token(token):
     _GitHubToken.value = token
+    print_github_token_rate_limit()
+
+
+def print_github_token_rate_limit():
+    github_token = get_github_token()
+    headers = {"Accept": "application/vnd.github.v3+json"}
+    if github_token:
+        headers["Authorization"] = f"token {github_token}"
+    res = requests.get(" https://api.github.com/rate_limit", headers=headers, timeout=1)
+    logger.warning("github rate limits: %s", res.json())

ccb/upstream_project.py

@@ -19,7 +19,7 @@
 )
 from .subprocess import check_output, check_call
 from .utils import SemaphoneStorage, format_duration
-from .github import get_github_token
+from .github import get_github_token, print_github_token_rate_limit
 
 
 logger = logging.getLogger(__name__)
@@ -236,6 +236,7 @@ async def versions(self):
                 headers = {"Accept": "application/vnd.github.v3+json"}
                 if github_token:
                     headers["Authorization"] = f"token {github_token}"
+                headers["User-Agent"] = "conan-center-bot qchateau"
                 async with aiohttp.ClientSession(raise_for_status=True, headers=headers) as client:
                     async with client.get(
                         f"https://api.github.com/repos/{self.owner}/{self.repo}/releases"
@@ -263,7 +264,14 @@ def _get_url_for_version(v):
                                 continue
                             r.url = _get_url_for_version(v)
                             self.__versions.append(r)
-
+
+            except aiohttp.ClientResponseError as exc:
+                logger.info("%s: error parsing repository: %s", self.recipe.name, exc)
+                for h in exc.headers:
+                    logger.info("%s: header %s=%s", self.recipe.name, h, exc.headers[h])
+                if exc.status == 403:
+                    print_github_token_rate_limit()
+                self.__versions = []                        
             except Exception as exc:
                 logger.info("%s: error parsing repository: %s", self.recipe.name, exc)
                 logger.debug(traceback.format_exc())
@@ -340,6 +348,11 @@ def _get_url_for_version(v):
                                 continue
                             r.url = _get_url_for_version(v)
                             self.__versions.append(r)
+            except aiohttp.ClientResponseError as exc:
+                logger.info("%s: error parsing repository: %s", self.recipe.name, exc)
+                if exc.status == 429:
+                    logger.warning("gitlab rate limited %s", exc.headers)
+                self.__versions = []                   
             except Exception as exc:
                 logger.info("%s: error parsing repository: %s", self.recipe.name, exc)
                 logger.debug(traceback.format_exc())