diff --git a/boofilsic/settings.py b/boofilsic/settings.py index 233efbd5..1047b79b 100644 --- a/boofilsic/settings.py +++ b/boofilsic/settings.py @@ -102,7 +102,7 @@ # The Movie Database (TMDB) - https://developer.themoviedb.org/ TMDB_API_V3_KEY=(str, "TESTONLY"), # Google Books - https://developers.google.com/books/docs/v1/using - not used at the moment - GOOGLE_API_KEY=(str, "TESTONLY"), + GOOGLE_API_KEY=(str, ""), # Discogs - personal access token from https://www.discogs.com/settings/developers DISCOGS_API_KEY=(str, "TESTONLY"), # IGDB - https://api-docs.igdb.com/ @@ -272,7 +272,7 @@ SPOTIFY_CREDENTIAL = env("SPOTIFY_API_KEY") TMDB_API3_KEY = env("TMDB_API_V3_KEY") # TMDB_API4_KEY = env('TMDB_API_V4_KEY') -# GOOGLE_API_KEY = env('GOOGLE_API_KEY') +GOOGLE_API_KEY = env("GOOGLE_API_KEY") DISCOGS_API_KEY = env("DISCOGS_API_KEY") IGDB_CLIENT_ID = env("IGDB_API_CLIENT_ID") IGDB_CLIENT_SECRET = env("IGDB_API_CLIENT_SECRET") diff --git a/catalog/common/downloaders.py b/catalog/common/downloaders.py index 341ca00e..38801ecf 100644 --- a/catalog/common/downloaders.py +++ b/catalog/common/downloaders.py @@ -22,6 +22,7 @@ RESPONSE_INVALID_CONTENT = -1 # content not valid but no need to retry RESPONSE_NETWORK_ERROR = -2 # network error, retry next proxied url RESPONSE_CENSORSHIP = -3 # censored, try sth special if possible +RESPONSE_QUOTA_EXCEEDED = -4 _mock_mode = False @@ -123,6 +124,8 @@ def __init__(self, downloader, msg=None): error = "Network Error" elif downloader.response_type == RESPONSE_CENSORSHIP: error = "Censored Content" + elif downloader.response_type == RESPONSE_QUOTA_EXCEEDED: + error = "API Quota Exceeded" else: error = "Unknown Error" self.message = ( @@ -169,6 +172,8 @@ def validate_response(self, response) -> int: return RESPONSE_NETWORK_ERROR elif response.status_code == 200: return RESPONSE_OK + elif response.status_code == 429: + return RESPONSE_QUOTA_EXCEEDED else: return RESPONSE_INVALID_CONTENT @@ -214,14 +219,6 @@ def download(self): class BasicDownloader2(BasicDownloader): - def validate_response(self, response) -> int: - if response is None: - return RESPONSE_NETWORK_ERROR - elif response.status_code == 200: - return RESPONSE_OK - else: - return RESPONSE_INVALID_CONTENT - def _download(self, url): try: if not _mock_mode: diff --git a/catalog/sites/google_books.py b/catalog/sites/google_books.py index 46a9efbb..dfcee10f 100644 --- a/catalog/sites/google_books.py +++ b/catalog/sites/google_books.py @@ -1,6 +1,9 @@ import logging import re +from django.conf import settings + +from catalog.book.utils import isbn_10_to_13 from catalog.common import * from catalog.models import * @@ -25,6 +28,8 @@ def id_to_url(cls, id_value): def scrape(self): api_url = f"https://www.googleapis.com/books/v1/volumes/{self.id_value}" + if settings.GOOGLE_API_KEY: + api_url += f"?key={settings.GOOGLE_API_KEY}" b = BasicDownloader(api_url).download().json() other = {} title = b["volumeInfo"]["title"] @@ -77,7 +82,7 @@ def scrape(self): isbn10 = iid["identifier"] if iid["type"] == "ISBN_13": isbn13 = iid["identifier"] - isbn = isbn13 # if isbn13 is not None else isbn10 + isbn = isbn13 if isbn13 is not None else isbn_10_to_13(isbn10) raw_img, ext = BasicImageDownloader.download_image(img_url, None, headers={}) data = { diff --git a/catalog/sites/spotify.py b/catalog/sites/spotify.py index da17ff35..5cea0cc7 100644 --- a/catalog/sites/spotify.py +++ b/catalog/sites/spotify.py @@ -41,7 +41,10 @@ def id_to_url(cls, id_value): def scrape(self): api_url = f"https://api.spotify.com/v1/albums/{self.id_value}" - headers = {"Authorization": f"Bearer {get_spotify_token()}"} + headers = { + "Authorization": f"Bearer {get_spotify_token()}", + "User-Agent": settings.NEODB_USER_AGENT, + } res_data = BasicDownloader(api_url, headers=headers).download().json() artist = [] for artist_dict in res_data["artists"]: