Skip to content

Commit

Permalink
Try to fix the API request errors in CI (#180)
Browse files Browse the repository at this point in the history
* Add sleep after 400 requests to Plausible

* Add more options for YouTube option
  • Loading branch information
bebatut authored Sep 30, 2024
1 parent 2d9eba9 commit d34a91a
Showing 1 changed file with 13 additions and 1 deletion.
14 changes: 13 additions & 1 deletion sources/bin/extract_gtn_tutorials.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env python

import argparse
import time
from datetime import date
from typing import (
Dict,
Expand All @@ -12,6 +13,8 @@
import yt_dlp
from owlready2 import get_ontology

PLAUSIBLE_REQUEST_NB = 0


def add_supported_servers(tuto: dict) -> None:
"""
Expand Down Expand Up @@ -80,8 +83,13 @@ def get_visit_results(url: str, tuto: dict, plausible_api: str) -> None:
"""
Extract visit results from Plausible URL
"""
global PLAUSIBLE_REQUEST_NB
headers = {"Authorization": f"Bearer {plausible_api}"}
if PLAUSIBLE_REQUEST_NB > 400:
time.sleep(3600)
PLAUSIBLE_REQUEST_NB = 0
results = shared.get_request_json(url, headers)
PLAUSIBLE_REQUEST_NB += 1
if "results" in results:
for metric in ["visitors", "pageviews", "visit_duration"]:
tuto[metric] += results["results"][metric]["value"]
Expand Down Expand Up @@ -109,7 +117,11 @@ def get_youtube_stats(tuto: dict) -> None:
"""
tuto["video_versions"] = 0
tuto["video_view"] = 0
ydl_opts = {"ignoreerrors": True, "quiet": True}
ydl_opts = {
"ignoreerrors": True,
"quiet": True,
"skip_download": True,
}
recordings = []
if "recordings" in tuto and tuto["recordings"]:
recordings = tuto["recordings"]
Expand Down

0 comments on commit d34a91a

Please sign in to comment.